DBIS
/
HyriseSQLParser
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactored test suite. added alias to expressions

This commit is contained in:
Pedro 2014-11-12 10:43:10 +01:00
parent c8e84d8f6a
commit 71cbe1bfac
12 changed files with 137 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
build/
utils/
# Compiled Object files

0
src/bin/.gitignore → bin/.gitignore vendored
View File

5
src/deploy_to_hyrise.sh → deploy_to_hyrise.sh
View File

@ -2,7 +2,8 @@
# Usage: deploy_to_hyrise.sh path/to/hyrise.git
BUILD_PATH=$(readlink -f $(dirname $0))/build
BUILD_PATH=$(readlink -f $(dirname $0))/src/build
HYRISE_PATH=$1
@ -14,7 +15,7 @@ if [ ! -d $SQL_PATH ]; then
fi
make build
make -C src/ build
rm ${SQL_PATH}/parser/*
cp ${BUILD_PATH}/* ${SQL_PATH}/parser/

20
run_tests.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
make clean -C src/
# make tests
make analysis -C src/
make grammar_test -C src/
echo "\n\n"
./bin/analysis "SELECT t1.a AS id, t1.b, t2.c FROM \"tbl\" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5"
# ./bin/analysis "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'"
echo "\n\n"
./bin/grammar_test -f "test/valid_queries.sql"
echo "\n\n"

29
src/Makefile
View File

@ -1,34 +1,31 @@
# Makefile
SOURCES = $(shell find lib/ -name '*.cpp') parser/bison_parser.cpp parser/flex_lexer.cpp parser/SQLParser.cpp
TESTS_MAIN = sql_tests.cpp
TESTS_BIN = bin/tests
CC = g++
CFLAGS = -g -O3 -Ilib/ -I./ -Iparser/ -std=c++11 -pthread
CFLAGS = -O3 -Ilib/ -I./ -Iparser/ -std=c++11 -pthread
SOURCES = $(shell find lib/ -name '*.cpp') parser/bison_parser.cpp parser/flex_lexer.cpp parser/SQLParser.cpp
BUILD_DIR = ../build/
BIN_DIR = ../bin
# release build is always using bison
build: clean
make -C parser/
mkdir build/
cp lib/* build/
cp parser/*.h build/
cp parser/*.cpp build/
mkdir $(BUILD_DIR)
cp lib/* $(BUILD_DIR)
cp parser/*.h $(BUILD_DIR)
cp parser/*.cpp $(BUILD_DIR)
analysis: $(SOURCES) sql_analysis.cpp
$(CC) $(CFLAGS) $(SOURCES) sql_analysis.cpp -o bin/analysis
$(CC) $(CFLAGS) $(SOURCES) sql_analysis.cpp -o $(BIN_DIR)/analysis
grammar_test: $(SOURCES) sql_grammar_test.cpp
$(CC) $(CFLAGS) $(SOURCES) sql_grammar_test.cpp -o bin/grammar_test
$(CC) $(CFLAGS) $(SOURCES) sql_grammar_test.cpp -o $(BIN_DIR)/grammar_test
tests: $(SOURCES) $(TESTS_MAIN)
$(CC) $(CFLAGS) $(SOURCES) $(TESTS_MAIN) -o $(TESTS_BIN)
$(CC) $(CFLAGS) $(SOURCES) sql_tests.cpp -o $(BIN_DIR)/tests
parser/bison_parser.cpp:
@ -36,6 +33,6 @@ parser/bison_parser.cpp:
clean:
rm -f *.o *~ bin/analysis $(TESTS_BIN) bin/grammar_test
rm -rf build/
rm -f *.o *~ $(BIN_DIR)/analysis $(TESTS_BIN) $(BIN_DIR)/grammar_test
rm -rf $(BUILD_DIR)
make clean -C parser/

39
src/build_and_run_tests.sh
View File

@ -1,39 +0,0 @@
make clean
# make tests
# ./bin/tests
make analysis
make grammar_test
echo "\n\n"
./bin/grammar_test "SELECT a FROM foo WHERE a > 12 OR b > 3 AND NOT c LIMIT 10"
./bin/grammar_test "SELECT col1, col2, 'test' FROM \"table\", foo AS t WHERE age > 12 AND zipcode = 12345 GROUP BY col1;"
./bin/grammar_test "SELECT * from \"table\" JOIN table2 ON a = b WHERE (b OR NOT a) AND a = 12.5"
./bin/grammar_test "(SELECT a FROM foo WHERE a > 12 OR b > 3 AND c NOT LIKE 's%' LIMIT 10);"
./bin/grammar_test "SELECT t1.a, t1.b, t2.c FROM \"table\" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5"
./bin/grammar_test "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'"
# Error: Where clause in between join statement
# ./bin/grammar_test -f "SELECT * from \"table\" WHERE (b OR NOT a) AND a = 12.5 AS t1 JOIN table2 ON a = b"
# ./bin/grammar_test -f "SELECT * \"table\" WHERE (b OR NOT a) AND a = 12.5 AS t1 JOIN table2 ON a = b"
echo "\n\n"
# ./bin/analysis "SELECT a FROM foo WHERE a > 12 OR b > 3 AND c = 3"
# ./bin/analysis "SELECT \"AVG(grade)\" FROM (SELECT AVG(grade) FROM students GROUP BY city) t1"
# ./bin/analysis "SELECT col1, col2, 'test' FROM tbl t1, foo WHERE age > 12 AND zipcode = 12345 GROUP BY col1 ORDER BY col2 DESC LIMIT 100;"
# ./bin/analysis "SELECT * from tbl AS t1 JOIN table2 AS t2 ON t1.a = t2.b WHERE (b OR NOT a) AND a = 12.5"
# ./bin/analysis "SELECT t1.a, t1.b, t2.c FROM tbl AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5"
# ./bin/analysis "-- test
# SELECT * FROM \"table\" WHERE a NOT LIKE '%s' -- inline comment
# --my comment"
# ./bin/analysis "
# IMPORT FROM TBL FILE 'students.tbl' INTO tbl;
# SELECT * FROM tbl;"
./bin/analysis "CREATE TABLE \"table\" FROM TBL FILE 'students.tbl'"
echo "\n\n"

3
src/lib/Expr.h
View File

@ -62,6 +62,7 @@ struct Expr {
Expr* expr2;
char* name;
char* table;
char* alias;
float fval;
int64_t ival;
@ -89,7 +90,7 @@ struct Expr {
Expr zero = {type}; \
var = (Expr*)malloc(sizeof *var); \
*var = zero; \
} while(0)
} while(0);
} // namespace hsql

5
src/lib/sqlhelper.cpp
View File

@ -67,7 +67,10 @@ void printExpression(Expr* expr, uint num_indent) {
case kExprLiteralString: inprint(expr->name, num_indent); break;
case kExprFunctionRef: inprint(expr->name, num_indent); inprint(expr->expr->name, num_indent+1); break;
case kExprOperator: printOperatorExpression(expr, num_indent); break;
default: fprintf(stderr, "Unrecognized expression type %d\n", expr->type); break;
default: fprintf(stderr, "Unrecognized expression type %d\n", expr->type); return;
}
if (expr->alias != NULL) {
inprint("Alias", num_indent); inprint(expr->alias, num_indent+1);
}
}

28
src/parser/bison_parser.y
View File

@ -99,7 +99,7 @@ typedef void* yyscan_t;
/*********************************
** Token Definition
*********************************/
%token <sval> NAME STRING COMPARISON
%token <sval> IDENTIFIER STRING
%token <fval> FLOAT
%token <ival> INT
%token <uval> NOTEQUALS LESSEQ GREATEREQ
@ -124,7 +124,7 @@ typedef void* yyscan_t;
%type <sval> table_name opt_alias alias file_path
%type <table> from_clause table_ref table_ref_atomic table_ref_name
%type <table> join_clause join_table
%type <expr> expr scalar_expr unary_expr binary_expr function_expr star_expr
%type <expr> expr scalar_expr unary_expr binary_expr function_expr star_expr expr_alias
%type <expr> column_name literal int_literal num_literal string_literal
%type <expr> comp_expr where_clause join_condition
%type <expr_list> expr_list group_clause select_list
@ -283,10 +283,16 @@ limit_clause:
** Expressions
******************************/
expr_list:
expr { $$ = new List<Expr*>($1); }
| expr_list ',' expr { $1->push_back($3); $$ = $1; }
expr_alias { $$ = new List<Expr*>($1); }
| expr_list ',' expr_alias { $1->push_back($3); $$ = $1; }
;
expr_alias:
expr opt_alias {
$$ = $1;
$$->alias = $2;
}
;
expr:
'(' expr ')' { $$ = $2; }
@ -332,12 +338,12 @@ comp_expr:
;
function_expr:
NAME '(' expr ')' { $$ = Expr::makeFunctionRef($1, $3); }
IDENTIFIER '(' expr ')' { $$ = Expr::makeFunctionRef($1, $3); }
;
column_name:
NAME { $$ = Expr::makeColumnRef($1); }
| NAME '.' NAME { $$ = Expr::makeColumnRef($1, $3); }
IDENTIFIER { $$ = Expr::makeColumnRef($1); }
| IDENTIFIER '.' IDENTIFIER { $$ = Expr::makeColumnRef($1, $3); }
;
literal:
@ -406,14 +412,14 @@ table_ref_name:
;
table_name:
NAME
| NAME '.' NAME
IDENTIFIER
| IDENTIFIER '.' IDENTIFIER
;
alias:
AS NAME { $$ = $2; }
| NAME
AS IDENTIFIER { $$ = $2; }
| IDENTIFIER
;
opt_alias:

4
src/parser/flex_lexer.l
View File

@ -131,12 +131,12 @@ IS TOKEN(IS)
\"[^\"\n]+\" {
// Crop the leading and trailing quote char
yylval->sval = hsql::substr(yytext, 1, strlen(yytext)-1);
return SQL_NAME;
return SQL_IDENTIFIER;
}
[A-Za-z][A-Za-z0-9_]* {
yylval->sval = strdup(yytext);
return SQL_NAME;
return SQL_IDENTIFIER;
}

79
src/sql_grammar_test.cpp
View File

@ -2,44 +2,95 @@
#include <stdio.h>
#include <string>
#include <chrono>
#include <fstream>
#include <sstream>
#include <vector>
#include "SQLParser.h"
using namespace hsql;
std::vector<std::string> readlines(std::string path) {
std::ifstream infile(path);
std::vector<std::string> lines;
std::string line;
while (std::getline(infile, line)) {
std::istringstream iss(line);
// Skip comments
if (line[0] != '#') {
lines.push_back(line);
}
}
return lines;
}
#define STREQ(a, b) (std::string(a).compare(std::string(b)) == 0)
int main(int argc, char *argv[]) {
if (argc <= 1) {
fprintf(stderr, "No SQL-Statement given!\n");
fprintf(stderr, "Usage: grammar_test [--false] [-f path] query, ...\n");
return -1;
}
bool expectFalse = (std::string("-f").compare(std::string(argv[1])) == 0);
bool expect_false = false;
bool use_file = false;
char* file_path;
int n = (expectFalse) ? 2 : 1;
for (; n < argc; ++n) {
char* sql = argv[n];
// Parse command line arguments
int i = 1;
for (; i < argc; ++i) {
if (STREQ(argv[i], "--false")) expect_false = true;
else if (STREQ(argv[i], "-f")) {
use_file = true;
file_path = argv[++i];
} else {
break;
}
}
// Read list of queries for this rest
std::vector<std::string> queries;
if (use_file) {
queries = readlines(file_path);
} else {
for (; i < argc; ++i) queries.push_back(argv[i]);
}
// Execute queries
int num_failed = 0;
for (std::string sql : queries) {
// Measuring the parsing time
std::chrono::time_point<std::chrono::system_clock> start, end;
start = std::chrono::system_clock::now();
// Parsing
StatementList* stmt_list = SQLParser::parseSQLString(sql);
StatementList* stmt_list = SQLParser::parseSQLString(sql.c_str());
end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end-start;
if (expectFalse == stmt_list->isValid) {
fprintf(stderr, "-> Failed (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql);
continue;
if (expect_false == stmt_list->isValid) {
fprintf(stderr, "Parsing failed (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql.c_str());
num_failed++;
} else {
if (expectFalse) {
printf("Success (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql);
if (expect_false) {
// printf("Success (%.3fms)! %s: \"%s\"\n", elapsed_seconds.count()*1000, stmt_list->parser_msg, sql.c_str());
} else {
// printf("Success (%.3fms)! \"%s\"\n", elapsed_seconds.count()*1000, sql.c_str());
}
}
}
if (num_failed == 0) {
printf("All %lu grammar tests completed successfully!", queries.size());
} else {
printf("Success (%.3fms)! \"%s\"\n", elapsed_seconds.count()*1000, sql);
}
}
fprintf(stderr, "Some grammar tests failed! %d out of %lu tests failed!", num_failed, queries.size());
}
return 0;
}

11
test/valid_queries.sql Normal file
View File

@ -0,0 +1,11 @@
# SELECT statement
SELECT * FROM orders;
SELECT a FROM foo WHERE a > 12 OR b > 3 AND NOT c LIMIT 10
SELECT col1 AS myname, col2, 'test' FROM "table", foo AS t WHERE age > 12 AND zipcode = 12345 GROUP BY col1;
SELECT * from "table" JOIN table2 ON a = b WHERE (b OR NOT a) AND a = 12.5
(SELECT a FROM foo WHERE a > 12 OR b > 3 AND c NOT LIKE 's%' LIMIT 10);
SELECT t1.a, t1.b, t2.c FROM "table" AS t1 JOIN (SELECT * FROM foo JOIN bar ON foo.id = bar.id) t2 ON t1.a = t2.b WHERE (t1.b OR NOT t1.a) AND t2.c = 12.5
# CREATE statement
CREATE TABLE "table" FROM TBL FILE 'students.tbl'
# Multiple statements
CREATE TABLE "table" FROM TBL FILE 'students.tbl'; SELECT * FROM "table";