From 71cbe1bfac05d72e19bbe7c6e87da1bfea81c69c Mon Sep 17 00:00:00 2001 From: Pedro Date: Wed, 12 Nov 2014 10:43:10 +0100 Subject: [PATCH] refactored test suite. added alias to expressions --- .gitignore | 1 + {src/bin => bin}/.gitignore | 0 ...deploy_to_hyrise.sh => deploy_to_hyrise.sh | 5 +- run_tests.sh | 20 +++++ src/Makefile | 29 ++++--- src/build_and_run_tests.sh | 39 ---------- src/lib/Expr.h | 3 +- src/lib/sqlhelper.cpp | 5 +- src/parser/bison_parser.y | 28 ++++--- src/parser/flex_lexer.l | 4 +- src/sql_grammar_test.cpp | 77 +++++++++++++++---- test/valid_queries.sql | 11 +++ 12 files changed, 137 insertions(+), 85 deletions(-) rename {src/bin => bin}/.gitignore (100%) rename src/deploy_to_hyrise.sh => deploy_to_hyrise.sh (79%) create mode 100755 run_tests.sh delete mode 100644 src/build_and_run_tests.sh create mode 100644 test/valid_queries.sql diff --git a/.gitignore b/.gitignore index c5d7a60..330b188 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +build/ utils/ # Compiled Object files diff --git a/src/bin/.gitignore b/bin/.gitignore similarity index 100% rename from src/bin/.gitignore rename to bin/.gitignore diff --git a/src/deploy_to_hyrise.sh b/deploy_to_hyrise.sh similarity index 79% rename from src/deploy_to_hyrise.sh rename to deploy_to_hyrise.sh index deaddab..fa54ce7 100755 --- a/src/deploy_to_hyrise.sh +++ b/deploy_to_hyrise.sh @@ -2,7 +2,8 @@ # Usage: deploy_to_hyrise.sh path/to/hyrise.git -BUILD_PATH=$(readlink -f $(dirname $0))/build + +BUILD_PATH=$(readlink -f $(dirname $0))/src/build HYRISE_PATH=$1 @@ -14,7 +15,7 @@ if [ ! -d $SQL_PATH ]; then fi -make build +make -C src/ build rm ${SQL_PATH}/parser/* cp ${BUILD_PATH}/* ${SQL_PATH}/parser/ diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..15ee6e3 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,20 @@ +#!/bin/sh + + +make clean -C src/ + +# make tests +make analysis -C src/ +make grammar_test -C src/ + +echo "\n\n" + + +./bin/analysis "SELECT t1.a AS id, t1.b, t2.c FROM \"tbl\" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5" +# ./bin/analysis "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'" + +echo "\n\n" + +./bin/grammar_test -f "test/valid_queries.sql" + +echo "\n\n" \ No newline at end of file diff --git a/src/Makefile b/src/Makefile index 24b3cc2..ce91070 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,34 +1,31 @@ -# Makefile - -SOURCES = $(shell find lib/ -name '*.cpp') parser/bison_parser.cpp parser/flex_lexer.cpp parser/SQLParser.cpp - -TESTS_MAIN = sql_tests.cpp -TESTS_BIN = bin/tests CC = g++ -CFLAGS = -g -O3 -Ilib/ -I./ -Iparser/ -std=c++11 -pthread +CFLAGS = -O3 -Ilib/ -I./ -Iparser/ -std=c++11 -pthread +SOURCES = $(shell find lib/ -name '*.cpp') parser/bison_parser.cpp parser/flex_lexer.cpp parser/SQLParser.cpp +BUILD_DIR = ../build/ +BIN_DIR = ../bin # release build is always using bison build: clean make -C parser/ - mkdir build/ - cp lib/* build/ - cp parser/*.h build/ - cp parser/*.cpp build/ + mkdir $(BUILD_DIR) + cp lib/* $(BUILD_DIR) + cp parser/*.h $(BUILD_DIR) + cp parser/*.cpp $(BUILD_DIR) analysis: $(SOURCES) sql_analysis.cpp - $(CC) $(CFLAGS) $(SOURCES) sql_analysis.cpp -o bin/analysis + $(CC) $(CFLAGS) $(SOURCES) sql_analysis.cpp -o $(BIN_DIR)/analysis grammar_test: $(SOURCES) sql_grammar_test.cpp - $(CC) $(CFLAGS) $(SOURCES) sql_grammar_test.cpp -o bin/grammar_test + $(CC) $(CFLAGS) $(SOURCES) sql_grammar_test.cpp -o $(BIN_DIR)/grammar_test tests: $(SOURCES) $(TESTS_MAIN) - $(CC) $(CFLAGS) $(SOURCES) $(TESTS_MAIN) -o $(TESTS_BIN) + $(CC) $(CFLAGS) $(SOURCES) sql_tests.cpp -o $(BIN_DIR)/tests parser/bison_parser.cpp: @@ -36,6 +33,6 @@ parser/bison_parser.cpp: clean: - rm -f *.o *~ bin/analysis $(TESTS_BIN) bin/grammar_test - rm -rf build/ + rm -f *.o *~ $(BIN_DIR)/analysis $(TESTS_BIN) $(BIN_DIR)/grammar_test + rm -rf $(BUILD_DIR) make clean -C parser/ diff --git a/src/build_and_run_tests.sh b/src/build_and_run_tests.sh deleted file mode 100644 index c92d588..0000000 --- a/src/build_and_run_tests.sh +++ /dev/null @@ -1,39 +0,0 @@ - -make clean - -# make tests -# ./bin/tests - -make analysis -make grammar_test - -echo "\n\n" - -./bin/grammar_test "SELECT a FROM foo WHERE a > 12 OR b > 3 AND NOT c LIMIT 10" -./bin/grammar_test "SELECT col1, col2, 'test' FROM \"table\", foo AS t WHERE age > 12 AND zipcode = 12345 GROUP BY col1;" -./bin/grammar_test "SELECT * from \"table\" JOIN table2 ON a = b WHERE (b OR NOT a) AND a = 12.5" -./bin/grammar_test "(SELECT a FROM foo WHERE a > 12 OR b > 3 AND c NOT LIKE 's%' LIMIT 10);" -./bin/grammar_test "SELECT t1.a, t1.b, t2.c FROM \"table\" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5" - -./bin/grammar_test "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'" - -# Error: Where clause in between join statement -# ./bin/grammar_test -f "SELECT * from \"table\" WHERE (b OR NOT a) AND a = 12.5 AS t1 JOIN table2 ON a = b" -# ./bin/grammar_test -f "SELECT * \"table\" WHERE (b OR NOT a) AND a = 12.5 AS t1 JOIN table2 ON a = b" - -echo "\n\n" - -# ./bin/analysis "SELECT a FROM foo WHERE a > 12 OR b > 3 AND c = 3" -# ./bin/analysis "SELECT \"AVG(grade)\" FROM (SELECT AVG(grade) FROM students GROUP BY city) t1" -# ./bin/analysis "SELECT col1, col2, 'test' FROM tbl t1, foo WHERE age > 12 AND zipcode = 12345 GROUP BY col1 ORDER BY col2 DESC LIMIT 100;" -# ./bin/analysis "SELECT * from tbl AS t1 JOIN table2 AS t2 ON t1.a = t2.b WHERE (b OR NOT a) AND a = 12.5" -# ./bin/analysis "SELECT t1.a, t1.b, t2.c FROM tbl AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5" -# ./bin/analysis "-- test -# SELECT * FROM \"table\" WHERE a NOT LIKE '%s' -- inline comment -# --my comment" -# ./bin/analysis " -# IMPORT FROM TBL FILE 'students.tbl' INTO tbl; -# SELECT * FROM tbl;" -./bin/analysis "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'" - -echo "\n\n" \ No newline at end of file diff --git a/src/lib/Expr.h b/src/lib/Expr.h index a2b07d8..32a9bd0 100644 --- a/src/lib/Expr.h +++ b/src/lib/Expr.h @@ -62,6 +62,7 @@ struct Expr { Expr* expr2; char* name; char* table; + char* alias; float fval; int64_t ival; @@ -89,7 +90,7 @@ struct Expr { Expr zero = {type}; \ var = (Expr*)malloc(sizeof *var); \ *var = zero; \ - } while(0) + } while(0); } // namespace hsql diff --git a/src/lib/sqlhelper.cpp b/src/lib/sqlhelper.cpp index 7a6a30f..688b779 100644 --- a/src/lib/sqlhelper.cpp +++ b/src/lib/sqlhelper.cpp @@ -67,7 +67,10 @@ void printExpression(Expr* expr, uint num_indent) { case kExprLiteralString: inprint(expr->name, num_indent); break; case kExprFunctionRef: inprint(expr->name, num_indent); inprint(expr->expr->name, num_indent+1); break; case kExprOperator: printOperatorExpression(expr, num_indent); break; - default: fprintf(stderr, "Unrecognized expression type %d\n", expr->type); break; + default: fprintf(stderr, "Unrecognized expression type %d\n", expr->type); return; + } + if (expr->alias != NULL) { + inprint("Alias", num_indent); inprint(expr->alias, num_indent+1); } } diff --git a/src/parser/bison_parser.y b/src/parser/bison_parser.y index 4d5ee5d..d381160 100644 --- a/src/parser/bison_parser.y +++ b/src/parser/bison_parser.y @@ -99,7 +99,7 @@ typedef void* yyscan_t; /********************************* ** Token Definition *********************************/ -%token NAME STRING COMPARISON +%token IDENTIFIER STRING %token FLOAT %token INT %token NOTEQUALS LESSEQ GREATEREQ @@ -124,7 +124,7 @@ typedef void* yyscan_t; %type table_name opt_alias alias file_path %type from_clause table_ref table_ref_atomic table_ref_name %type
join_clause join_table -%type expr scalar_expr unary_expr binary_expr function_expr star_expr +%type expr scalar_expr unary_expr binary_expr function_expr star_expr expr_alias %type column_name literal int_literal num_literal string_literal %type comp_expr where_clause join_condition %type expr_list group_clause select_list @@ -283,10 +283,16 @@ limit_clause: ** Expressions ******************************/ expr_list: - expr { $$ = new List($1); } - | expr_list ',' expr { $1->push_back($3); $$ = $1; } + expr_alias { $$ = new List($1); } + | expr_list ',' expr_alias { $1->push_back($3); $$ = $1; } ; +expr_alias: + expr opt_alias { + $$ = $1; + $$->alias = $2; + } + ; expr: '(' expr ')' { $$ = $2; } @@ -332,12 +338,12 @@ comp_expr: ; function_expr: - NAME '(' expr ')' { $$ = Expr::makeFunctionRef($1, $3); } + IDENTIFIER '(' expr ')' { $$ = Expr::makeFunctionRef($1, $3); } ; column_name: - NAME { $$ = Expr::makeColumnRef($1); } - | NAME '.' NAME { $$ = Expr::makeColumnRef($1, $3); } + IDENTIFIER { $$ = Expr::makeColumnRef($1); } + | IDENTIFIER '.' IDENTIFIER { $$ = Expr::makeColumnRef($1, $3); } ; literal: @@ -406,14 +412,14 @@ table_ref_name: ; table_name: - NAME - | NAME '.' NAME + IDENTIFIER + | IDENTIFIER '.' IDENTIFIER ; alias: - AS NAME { $$ = $2; } - | NAME + AS IDENTIFIER { $$ = $2; } + | IDENTIFIER ; opt_alias: diff --git a/src/parser/flex_lexer.l b/src/parser/flex_lexer.l index 6b9f4d3..aec4e0a 100644 --- a/src/parser/flex_lexer.l +++ b/src/parser/flex_lexer.l @@ -131,12 +131,12 @@ IS TOKEN(IS) \"[^\"\n]+\" { // Crop the leading and trailing quote char yylval->sval = hsql::substr(yytext, 1, strlen(yytext)-1); - return SQL_NAME; + return SQL_IDENTIFIER; } [A-Za-z][A-Za-z0-9_]* { yylval->sval = strdup(yytext); - return SQL_NAME; + return SQL_IDENTIFIER; } diff --git a/src/sql_grammar_test.cpp b/src/sql_grammar_test.cpp index 27f3aa6..667f300 100644 --- a/src/sql_grammar_test.cpp +++ b/src/sql_grammar_test.cpp @@ -2,44 +2,95 @@ #include #include #include +#include +#include +#include #include "SQLParser.h" using namespace hsql; + +std::vector readlines(std::string path) { + std::ifstream infile(path); + std::vector lines; + std::string line; + while (std::getline(infile, line)) { + std::istringstream iss(line); + + // Skip comments + if (line[0] != '#') { + lines.push_back(line); + } + } + return lines; +} + + +#define STREQ(a, b) (std::string(a).compare(std::string(b)) == 0) + int main(int argc, char *argv[]) { if (argc <= 1) { - fprintf(stderr, "No SQL-Statement given!\n"); + fprintf(stderr, "Usage: grammar_test [--false] [-f path] query, ...\n"); return -1; } - bool expectFalse = (std::string("-f").compare(std::string(argv[1])) == 0); + bool expect_false = false; + bool use_file = false; + char* file_path; + + // Parse command line arguments + int i = 1; + for (; i < argc; ++i) { + if (STREQ(argv[i], "--false")) expect_false = true; + else if (STREQ(argv[i], "-f")) { + use_file = true; + file_path = argv[++i]; + } else { + break; + } + } - int n = (expectFalse) ? 2 : 1; - for (; n < argc; ++n) { - char* sql = argv[n]; + // Read list of queries for this rest + std::vector queries; + if (use_file) { + queries = readlines(file_path); + } else { + for (; i < argc; ++i) queries.push_back(argv[i]); + } + + + // Execute queries + int num_failed = 0; + for (std::string sql : queries) { // Measuring the parsing time std::chrono::time_point start, end; start = std::chrono::system_clock::now(); // Parsing - StatementList* stmt_list = SQLParser::parseSQLString(sql); + StatementList* stmt_list = SQLParser::parseSQLString(sql.c_str()); end = std::chrono::system_clock::now(); std::chrono::duration elapsed_seconds = end-start; - if (expectFalse == stmt_list->isValid) { - fprintf(stderr, "-> Failed (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql); - continue; + if (expect_false == stmt_list->isValid) { + fprintf(stderr, "Parsing failed (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql.c_str()); + num_failed++; } else { - if (expectFalse) { - printf("Success (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql); - + if (expect_false) { + // printf("Success (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql.c_str()); } else { - printf("Success (%.3fms)! \"%s\"\n", elapsed_seconds.count()*1000, sql); + // printf("Success (%.3fms)! \"%s\"\n", elapsed_seconds.count()*1000, sql.c_str()); } } } + if (num_failed == 0) { + printf("All %lu grammar tests completed successfully!", queries.size()); + } else { + fprintf(stderr, "Some grammar tests failed! %d out of %lu tests failed!", num_failed, queries.size()); + } + + return 0; } \ No newline at end of file diff --git a/test/valid_queries.sql b/test/valid_queries.sql new file mode 100644 index 0000000..8960b00 --- /dev/null +++ b/test/valid_queries.sql @@ -0,0 +1,11 @@ +# SELECT statement +SELECT * FROM orders; +SELECT a FROM foo WHERE a > 12 OR b > 3 AND NOT c LIMIT 10 +SELECT col1 AS myname, col2, 'test' FROM "table", foo AS t WHERE age > 12 AND zipcode = 12345 GROUP BY col1; +SELECT * from "table" JOIN table2 ON a = b WHERE (b OR NOT a) AND a = 12.5 +(SELECT a FROM foo WHERE a > 12 OR b > 3 AND c NOT LIKE 's%' LIMIT 10); +SELECT t1.a, t1.b, t2.c FROM "table" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5 +# CREATE statement +CREATE TABLE "table" FROM TBL FILE 'students.tbl' +# Multiple statements +CREATE TABLE "table" FROM TBL FILE 'students.tbl'; SELECT * FROM "table"; \ No newline at end of file