diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index 2cc78ebc..e3a1cc91 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -235,8 +235,14 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& properties += "]"; std::string params; params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer); + params += std::format("stopWords := '{}', ", + ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + params += std::format("ignore_pattern := '{}', ", + formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern)); params += - std::format("stopWords := '{}'", ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + std::format("tokenizer := '{}', ", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer); + params += std::format("jieba_dict_dir := '{}'", + formatStrInCypher(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir)); query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName, properties, params); query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName); diff --git a/fts/src/function/fts_config.cpp b/fts/src/function/fts_config.cpp index d3b7c7cf..b92d7846 100644 --- a/fts/src/function/fts_config.cpp +++ b/fts/src/function/fts_config.cpp @@ -145,11 +145,11 @@ CreateFTSConfig::CreateFTSConfig(main::ClientContext& context, common::table_id_ } else if (IgnorePattern::NAME == lowerCaseName) { value.validateType(IgnorePattern::TYPE); ignorePattern = common::StringUtils::getLower(value.getValue()); + // Wildcard characters ('*' and '?') are protected from the ignore pattern during + // query normalization (see FTSUtils::normalizeQuery), so the same pattern is used + // for indexing and for queries. ignorePatternQuery = ignorePattern; - common::StringUtils::replaceAll(ignorePatternQuery, "*", ""); - common::StringUtils::replaceAll(ignorePatternQuery, "?", ""); IgnorePattern::validate(ignorePattern); - IgnorePattern::validate(ignorePatternQuery); } else if (lowerCaseName == "tokenizer") { value.validateType(common::LogicalTypeID::STRING); tokenizerInfo.tokenizer = common::StringUtils::getLower(value.getValue()); diff --git a/fts/src/function/query_fts_bind_data.cpp b/fts/src/function/query_fts_bind_data.cpp index 1075c7da..1e19ad0c 100644 --- a/fts/src/function/query_fts_bind_data.cpp +++ b/fts/src/function/query_fts_bind_data.cpp @@ -47,7 +47,8 @@ std::vector QueryFTSBindData::getQueryTerms(main::ClientContext& co auto queryInStr = ExpressionUtil::evaluateLiteral(&context, query, LogicalType::STRING()); auto config = entry.getAuxInfo().cast().config; - FTSUtils::normalizeQuery(queryInStr, config.ignorePatternQuery); + FTSUtils::normalizeQuery(queryInStr, config.ignorePatternQuery, + true /* protectWildcardChars */); auto terms = FTSUtils::tokenizeString(queryInStr, config); auto stopWordsTable = StorageManager::Get(context) diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index 96ae0298..8c302d09 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -13,7 +13,8 @@ namespace fts_extension { struct FTSUtils { - static void normalizeQuery(std::string& query, const regex::RE2& ignorePattern); + static void normalizeQuery(std::string& query, const regex::RE2& ignorePattern, + bool protectWildcardChars = false); static bool hasWildcardPattern(const std::string& term); diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index 1eafaf59..652dd16e 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -1,5 +1,7 @@ #include "utils/fts_utils.h" +#include +#include #include #include "common/string_utils.h" @@ -18,10 +20,38 @@ using namespace lbug::storage; using namespace lbug::transaction; using namespace lbug::catalog; -void FTSUtils::normalizeQuery(std::string& query, const RE2& ignorePattern) { - std::string replacePattern = " "; - RE2::GlobalReplace(&query, ignorePattern, replacePattern); - StringUtils::toLower(query); +void FTSUtils::normalizeQuery(std::string& query, const RE2& ignorePattern, + bool protectWildcardChars) { + // Wildcard characters in the query must survive normalization, even if the ignore pattern + // would match them (e.g. a negated character class like [^[:alnum:]-]+ matches any + // non-alphanumeric character, including '*' and '?'). To achieve this, we normalize the + // query in segments between wildcard characters and re-attach the wildcard characters + // afterwards. Document contents are normalized without wildcard protection. + if (protectWildcardChars) { + std::string result; + std::string segment; + auto normalizeSegment = [&]() { + if (!segment.empty()) { + RE2::GlobalReplace(&segment, ignorePattern, " "); + result += segment; + segment.clear(); + } + }; + for (auto c : query) { + if (c == '*' || c == '?') { + normalizeSegment(); + result += c; + } else { + segment += c; + } + } + normalizeSegment(); + StringUtils::toLower(result); + query = std::move(result); + } else { + RE2::GlobalReplace(&query, ignorePattern, " "); + StringUtils::toLower(query); + } } struct StopWordsChecker { @@ -115,6 +145,12 @@ std::vector FTSUtils::tokenizeString(std::string& str, const FTSCon config.jiebaDictDir + "/hmm_model.utf8", config.jiebaDictDir + "/user.dict.utf8", config.jiebaDictDir + "/idf.utf8", config.jiebaDictDir + "/stop_words.utf8"); jieba.CutForSearch(str, terms); + // CutForSearch keeps the whitespace between words as separate tokens. Whitespace is + // never a meaningful term, so we skip those tokens. + std::erase_if(terms, [](const std::string& term) { + return std::all_of(term.begin(), term.end(), + [](unsigned char c) { return std::isspace(c); }); + }); } else { terms = StringUtils::split(str, " ", true /* ignoreEmptyStringParts */); } diff --git a/fts/test/test_files/ignore_pattern.test b/fts/test/test_files/ignore_pattern.test new file mode 100644 index 00000000..7cb8289a --- /dev/null +++ b/fts/test/test_files/ignore_pattern.test @@ -0,0 +1,50 @@ +-DATASET CSV fts-emails + +-- + +# Tests that a custom ignore_pattern passed to CREATE_FTS_INDEX is also used when +# normalizing the query in QUERY_FTS_INDEX (see issue #910). The custom pattern +# preserves digits and hyphens, so tokens like `allen-p` and `102` are stored as +# single terms in the index and must be searchable with exact queries. With the +# default ignore pattern, `allen-p` would be split into `allen` and `p` and the +# digits would be stripped from `102`. +-CASE ignorePatternQuery +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CALL CREATE_FTS_INDEX('emails', 'fileIdx', ['file'], stemmer := 'none', ignore_pattern := '[^[:alnum:]-]+'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen-p') RETURN count(*); +---- 1 +13 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen p') RETURN count(*); +---- 1 +0 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'sent mail') RETURN count(*); +---- 1 +13 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '102') RETURN node.id; +---- 1 +10 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '1004') RETURN node.id; +---- 1 +8 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'zzz') RETURN node.id; +---- 0 + +# Wildcard queries keep working with a custom ignore pattern. +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '10??') RETURN node.id ORDER BY node.id; +---- 5 +4 +5 +6 +7 +8 + +# The custom ignore_pattern must also be used after the index is reloaded from disk. +-RELOADDB +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', 'allen-p') RETURN count(*); +---- 1 +13 +-STATEMENT CALL QUERY_FTS_INDEX('emails', 'fileIdx', '102') RETURN node.id; +---- 1 +10