Commit d7a8c463 authored by Eric Jiang's avatar Eric Jiang
Browse files

Fix filtering TreeItem lists by non-ASCII strings

AssetFilter tries to normalize the TreeItem's text by removing any
punctuation and symbols, but it was done using a simple [^a-zA-Z0-9\s]
character class which removed any non-ASCII characters, breaking the
search for many non-English languages. Replacing this with [^\w\s]
didn't work, but iterating through the string and using
isLetterOrNumber() does work.

BUG: 432699
parent ebc087ad
Pipeline #209494 passed with stage
in 8 minutes and 59 seconds
......@@ -42,15 +42,27 @@ bool AssetFilter::filterName(const std::shared_ptr<TreeItem> &item) const
if (!m_name_enabled) {
return true;
QString itemId = item->dataColumn(AssetTreeModel::IdCol).toString().toUtf8().constData();
itemId = itemId.normalized(QString::NormalizationForm_D).remove(QRegularExpression(QStringLiteral("[^a-zA-Z0-9\\s]")));
const QString itemId = normalizeText(item->dataColumn(AssetTreeModel::IdCol).toString());
QString itemText = i18n(item->dataColumn(AssetTreeModel::NameCol).toString().toUtf8().constData());
itemText = itemText.normalized(QString::NormalizationForm_D).remove(QRegularExpression(QStringLiteral("[^a-zA-Z0-9\\s]")));
QString patt = m_name_value.normalized(QString::NormalizationForm_D).remove(QRegularExpression(QStringLiteral("[^a-zA-Z0-9\\s]")));
itemText = normalizeText(itemText);
QString patt = normalizeText(m_name_value);
return itemText.contains(patt, Qt::CaseInsensitive) || itemId.contains(patt, Qt::CaseInsensitive);
QString AssetFilter::normalizeText(const QString &text)
// NormalizationForm_D decomposes letters with diacritics, and then the
// diacritics are removed by checking isLetterOrNumber().
const QString normalized = text.normalized(QString::NormalizationForm_D);
QString newString;
std::copy_if(normalized.begin(), normalized.end(), std::back_inserter(newString),
[](QChar c){
return c.isLetterOrNumber() || c.isSpace();
return newString;
bool AssetFilter::filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const
QModelIndex row = sourceModel()->index(sourceRow, 0, sourceParent);
......@@ -43,7 +43,15 @@ public:
bool filterAcceptsRow(int sourceRow, const QModelIndex &sourceParent) const override;
bool lessThan(const QModelIndex &left, const QModelIndex &right) const override;
/** @brief Given a TreeItem, check if the ID and Name (first and second)
* columns contain the filtered name as a substring.
* @return true if either ID or Name contains the filtered name as a
* substring, false otherwise
bool filterName(const std::shared_ptr<TreeItem> &item) const;
/** @brief Returns a copy of the input string with any characters that are
* not letters, numbers, or spaces removed. */
static QString normalizeText(const QString &text);
/** @brief Apply all filter and returns true if the object should be kept after filtering */
virtual bool applyAll(std::shared_ptr<TreeItem> item) const;
......@@ -204,10 +204,11 @@ TEST_CASE("Effect filter text-matching logic")
item->setData(AssetTreeModel::IdCol, "Hello World");
item->setData(AssetTreeModel::NameCol, "This is K-denlive");
filter.setFilterName(true, "Worl");
filter.setFilterName(true, "wORL");
CHECK(filter.filterName(item) == true);
filter.setFilterName(true, "Abcd");
// should not search across spaces
filter.setFilterName(true, "Thisis");
CHECK(filter.filterName(item) == false);
// should ignore punctuation
......@@ -232,4 +233,17 @@ TEST_CASE("Effect filter text-matching logic")
filter.setFilterName(true, "ミュ");
CHECK(filter.filterName(item) == true);
SECTION("Ignores diacritics")
auto item = TreeItem::construct(rootData, model, true);
item->setData(AssetTreeModel::IdCol, "rgb parade");
item->setData(AssetTreeModel::NameCol, "Défilé RVB");
// should be able to search with and without diacritics
filter.setFilterName(true, "defile");
CHECK(filter.filterName(item) == true);
filter.setFilterName(true, "ilé");
CHECK(filter.filterName(item) == true);
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment