fix(fts): honor custom ignore_pattern (and tokenizer) when normalizing queries

fts/src/function/create_fts_index.cpp

@@ -235,8 +235,14 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
     properties += "]";
     std::string params;
     params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer);
+    params += std::format("stopWords := '{}', ",
+        ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords);
+    params += std::format("ignore_pattern := '{}', ",
+        formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern));
     params +=
-        std::format("stopWords := '{}'", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords);
+        std::format("tokenizer := '{}', ", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer);
+    params += std::format("jieba_dict_dir := '{}'",
+        formatStrInCypher(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir));
     query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName,
         properties, params);
     query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName);

fts/src/function/fts_config.cpp

@@ -145,11 +145,11 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_
         } else if (IgnorePattern::NAME == lowerCaseName) {
             value.validateType(IgnorePattern::TYPE);
             ignorePattern = common::StringUtils::getLower(value.getValue<std::string>());
+            // Wildcard characters ('*' and '?') are protected from the ignore pattern during
+            // query normalization (see FTSUtils::normalizeQuery), so the same pattern is used
+            // for indexing and for queries.
             ignorePatternQuery = ignorePattern;
-            common::StringUtils::replaceAll(ignorePatternQuery, "*", "");
-            common::StringUtils::replaceAll(ignorePatternQuery, "?", "");
             IgnorePattern::validate(ignorePattern);
-            IgnorePattern::validate(ignorePatternQuery);
         } else if (lowerCaseName == "tokenizer") {
             value.validateType(common::LogicalTypeID::STRING);
             tokenizerInfo.tokenizer = common::StringUtils::getLower(value.getValue<std::string>());

fts/src/function/query_fts_bind_data.cpp

@@ -47,7 +47,8 @@ std::vector<std::string> QueryFTSBindData::getQueryTerms(main::ClientContext& co
     auto queryInStr =
         ExpressionUtil::evaluateLiteral<std::string>(&context, query, LogicalType::STRING());
     auto config = entry.getAuxInfo().cast<FTSIndexAuxInfo>().config;
-    FTSUtils::normalizeQuery(queryInStr, config.ignorePatternQuery);
+    FTSUtils::normalizeQuery(queryInStr, config.ignorePatternQuery,
+        true /* protectWildcardChars */);
     auto terms = FTSUtils::tokenizeString(queryInStr, config);
     auto stopWordsTable =
         StorageManager::Get(context)

fts/src/include/utils/fts_utils.h

@@ -13,7 +13,8 @@ namespace fts_extension {
 
 struct FTSUtils {
 
-    static void normalizeQuery(std::string& query, const regex::RE2& ignorePattern);
+    static void normalizeQuery(std::string& query, const regex::RE2& ignorePattern,
+        bool protectWildcardChars = false);
 
     static bool hasWildcardPattern(const std::string& term);
 

fts/src/utils/fts_utils.cpp

@@ -1,5 +1,7 @@
 #include "utils/fts_utils.h"
 
+#include <algorithm>
+#include <cctype>
 #include <optional>
 
 #include "common/string_utils.h"
@@ -18,10 +20,38 @@ using namespace lbug::storage;
 using namespace lbug::transaction;
 using namespace lbug::catalog;
 
-void FTSUtils::normalizeQuery(std::string& query, const RE2& ignorePattern) {
-    std::string replacePattern = " ";
-    RE2::GlobalReplace(&query, ignorePattern, replacePattern);
-    StringUtils::toLower(query);
+void FTSUtils::normalizeQuery(std::string& query, const RE2& ignorePattern,
+    bool protectWildcardChars) {
+    // Wildcard characters in the query must survive normalization, even if the ignore pattern
+    // would match them (e.g. a negated character class like [^[:alnum:]-]+ matches any
+    // non-alphanumeric character, including '*' and '?'). To achieve this, we normalize the
+    // query in segments between wildcard characters and re-attach the wildcard characters
+    // afterwards. Document contents are normalized without wildcard protection.
+    if (protectWildcardChars) {
+        std::string result;
+        std::string segment;
+        auto normalizeSegment = [&]() {
+            if (!segment.empty()) {
+                RE2::GlobalReplace(&segment, ignorePattern, " ");
+                result += segment;
+                segment.clear();
+            }
+        };
+        for (auto c : query) {
+            if (c == '*' || c == '?') {
+                normalizeSegment();
+                result += c;
+            } else {
+                segment += c;
+            }
+        }
+        normalizeSegment();
+        StringUtils::toLower(result);
+        query = std::move(result);
+    } else {
+        RE2::GlobalReplace(&query, ignorePattern, " ");
+        StringUtils::toLower(query);
+    }
 }
 
 struct StopWordsChecker {
@@ -115,6 +145,12 @@ std::vector<std::string> FTSUtils::tokenizeString(std::string& str, const FTSCon
             config.jiebaDictDir + "/hmm_model.utf8", config.jiebaDictDir + "/user.dict.utf8",
             config.jiebaDictDir + "/idf.utf8", config.jiebaDictDir + "/stop_words.utf8");
         jieba.CutForSearch(str, terms);
+        // CutForSearch keeps the whitespace between words as separate tokens. Whitespace is
+        // never a meaningful term, so we skip those tokens.
+        std::erase_if(terms, [](const std::string& term) {
+            return std::all_of(term.begin(), term.end(),
+                [](unsigned char c) { return std::isspace(c); });
+        });
     } else {
         terms = StringUtils::split(str, " ", true /* ignoreEmptyStringParts */);
     }

fts/test/test_files/ignore_pattern.test

@@ -0,0 +1,50 @@
+-DATASET CSV fts-emails
+
+--
+
+# Tests that a custom ignore_pattern passed to CREATE_FTS_INDEX is also used when
+# normalizing the query in QUERY_FTS_INDEX (see issue #910). The custom pattern
+# preserves digits and hyphens, so tokens like `allen-p` and `102` are stored as
+# single terms in the index and must be searchable with exact queries. With the
+# default ignore pattern, `allen-p` would be split into `allen` and `p` and the
+# digits would be stripped from `102`.
+-CASE ignorePatternQuery
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CALL CREATE_FTS_INDEX('emails', 'fileIdx', ['file'], stemmer := 'none', ignore_pattern := '[^[:alnum:]-]+');
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen-p') RETURN count(*);
+---- 1
+13
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen p') RETURN count(*);
+---- 1
+0
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'sent mail') RETURN count(*);
+---- 1
+13
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '102') RETURN node.id;
+---- 1
+10
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '1004') RETURN node.id;
+---- 1
+8
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'zzz') RETURN node.id;
+---- 0
+
+# Wildcard queries keep working with a custom ignore pattern.
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '10??') RETURN node.id ORDER BY node.id;
+---- 5
+4
+5
+6
+7
+8
+
+# The custom ignore_pattern must also be used after the index is reloaded from disk.
+-RELOADDB
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen-p') RETURN count(*);
+---- 1
+13
+-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '102') RETURN node.id;
+---- 1
+10