extended flex to use custom token type

lemon_example/build_and_run.sh

@@ -1,4 +1,4 @@
-rm -f lemon_test lemon_parser.c flex_scanner.c
+rm -f lemon_test lemon_parser.c lemon_parser.h flex_scanner.c flex_scanner.h
 echo "Lemon"
 lemon lemon_parser.y
 echo "Flex"

lemon_example/flex_scanner.l

@@ -1,15 +1,20 @@
 %{
-    #include "lemon_parser.h"
-    #include <stdlib.h>
+
+#include "lemon_parser.h"
+#include <stdlib.h>
+#include "token_def.h"
+
+
 %}
 
 
 %option reentrant
 %option noyywrap
+%option bison-bridge
 
 
 %%
-[0-9]+|[0-9]+.[0-9]+      return NUMBER;
+[0-9]+|[0-9]+.[0-9]+      { yylval->fval = atof(yytext); return NUMBER; }
 [+]         return PLUS;
 [\n]        return NL;
 [ \t]       ;       /* skip whitespace */

lemon_example/lemon_parser.y

@@ -1,16 +1,16 @@
 %include {
 	#include <assert.h>
 	#include <cstdlib>
+	#include "token_def.h"
 }
 %syntax_error { printf("Lemon syntax error\n"); }
 
-%token_type {const char*}
-%type expr {float}
-
-%left PLUS MINUS .
-
+%token_type {LexerToken*}
 %extra_argument { float* result }
 
+%type expr {float}
+%left PLUS MINUS .
+
 start   ::= prog.
 
 prog    ::= prog print NL .
@@ -19,5 +19,5 @@ prog    ::= .
 
 print   ::= expr(a) .         { *result = a; }
 
-expr(a) ::= NUMBER(b) .                { a = atof(b); }
+expr(a) ::= NUMBER(b) .                { a = b->fval; }
 expr(a) ::= expr(b) PLUS expr(c) .      { a = b + c; }

lemon_example/main.cpp

@@ -1,8 +1,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <thread>
-#include "flex_scanner.h"
+#include "token_def.h"
 #include "lemon_parser.h"
+#include "flex_scanner.h"
 
 // Based on https://github.com/theory/flex-lemon-example
 // and http://stackoverflow.com/questions/24833465/bison-yacc-vs-lemon-vs-standard-input
@@ -10,10 +11,10 @@ typedef float ResultType;
 
 void *ParseAlloc(void *(*mallocProc)(size_t));
 void ParseFree(void *p, void (*freeProc)(void*));
-void Parse(void *yyp, int yymajor, const char* text, ResultType*);
+void Parse(void *yyp, int yymajor, LexerToken* token, ResultType*);
 
-int yylex(void);
-int yylval;
+// int yylex(void);
+// int yylval;
 
 
 float parseString(const char* string) {
@@ -27,15 +28,16 @@ float parseString(const char* string) {
 	int tokenCode;
 	ResultType result;
 	do {
-		tokenCode = yylex(scanner);
-		Parse(lemonParser, tokenCode, yyget_text(scanner), &result);
+		LexerToken token;
+		tokenCode = yylex(&token, scanner);
+		Parse(lemonParser, tokenCode, &token, &result);
 		// printf("Token %d\n", tokenCode);
 	} while (tokenCode > 0);
 
 	return result;
 }
 
-
+int parse_count = 0;
 void multithreadTest(int numOfParses, int id) {
 	for (int n = 0; n < numOfParses; ++n) {
 		int a = rand() % 1000 + 1;
@@ -44,14 +46,18 @@ void multithreadTest(int numOfParses, int id) {
 		char string[32];
 		sprintf(string, "%d + %d", a, b);
 
+		parse_count++;
 		int result = parseString(string);
+		if (parse_count != 1) printf("+");
+		parse_count--;
+
 		if (result != c) printf("Error[%d]! %s != %d\n", id, string, result);
 	}
 }
 
 int main(void) {
-	const int numThreads = 10;
-	int numRuns = 5000;
+	const int numThreads = 20;
+	int numRuns = 300;
 
 	std::thread threads[numThreads];
 	for (int n = 0; n < numThreads; ++n) {

lemon_example/token_def.h

@@ -0,0 +1,9 @@
+
+
+struct LexerToken {
+	char* sval;
+	float fval;
+};
+
+#define YYSTYPE LexerToken
+