Fix: Resolved deterministic memory leak and dangling pointer in SQLParser::tokenize

.github/workflows/ci.yml

@@ -46,9 +46,22 @@ jobs:
             cxx: clang++
             os: macos-latest
 
+          - name: clang-sanitizer-ubuntu
+            cc: clang-19
+            cxx: clang++-19
+            os: ubuntu-latest
+            container: ubuntu:24.04
+            build_options: "CXXFLAGS='-fsanitize=address,undefined -g' LDFLAGS='-fsanitize=address,undefined'"
+
+          - name: clang-sanitizer-macOS
+            cc: clang
+            cxx: clang++
+            os: macos-latest
+            build_options: "CXXFLAGS='-fsanitize=address,undefined -g' LDFLAGS='-fsanitize=address,undefined'"  
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         if: matrix.name != 'gcc-6'
 
       - name: Checkout (Ubuntu 18.04)
@@ -67,14 +80,14 @@ jobs:
           git checkout $GITHUB_HEAD_REF
 
       - name: Setup (macOS)
-        if: matrix.name == 'clang-macOS'
+        if: matrix.os == 'macos-latest'
         run: |
           brew install bison flex
           echo "BISON=$(brew --prefix bison)/bin/bison" >> $GITHUB_ENV
           echo "FLEX=$(brew --prefix flex)/bin/flex" >> $GITHUB_ENV
 
       - name: Setup (Ubuntu)
-        if: matrix.name != 'clang-macOS'
+        if: matrix.os == 'ubuntu-latest'
         run: |
           apt-get update
           apt-get install --no-install-recommends -y bison flex ${CC} ${CXX} make valgrind

src/SQLParser.cpp

@@ -59,11 +59,13 @@ bool SQLParser::tokenize(const std::string& sql, std::vector<int16_t>* tokens) {
   int16_t token = hsql_lex(&yylval, &yylloc, scanner);
   while (token != 0) {
     tokens->push_back(token);
-    token = hsql_lex(&yylval, &yylloc, scanner);
 
     if (token == SQL_IDENTIFIER || token == SQL_STRING) {
       free(yylval.sval);
+      yylval.sval = nullptr;
     }
+    token = hsql_lex(&yylval, &yylloc, scanner);
+
   }
 
   hsql__delete_buffer(state, scanner);

test/sql_parser.cpp

@@ -42,3 +42,17 @@ TEST(SQLParserTokenizeStringifyTest) {
   ASSERT(query == cache[token_string]);
   ASSERT(&query != &cache[token_string]);
 }
+
+TEST(SQLParserTokenizeLeakRegressionTest) {
+
+  const std::string query = "'string_1' 'string_2' 'string_3';";
+  std::vector<int16_t> tokens;
+
+  ASSERT(SQLParser::tokenize(query, &tokens));
+
+  ASSERT_EQ(tokens.size(), 4);
+  ASSERT_EQ(tokens[0], SQL_STRING);
+  ASSERT_EQ(tokens[1], SQL_STRING);
+  ASSERT_EQ(tokens[2], SQL_STRING);
+  ASSERT_EQ(tokens[3], ';');
+}