Make the code that removes items from KFileItemModel more robust

When we remove items from the model, we called the function
KFileItemModel::removeItems(const KFileItemList&, RemoveItemsBehavior).
This function then looked up the indexes of the items using the hash
m_items. This is wasteful in the situations when the indexes of the
removed items are known in advance (like when an expanded folder is
collapsed in Details View), and it can cause trouble if one item is
contained in the model multiple times (can happen when searching, and a
file both matches the search and is a child of a folder that matches
the search). Even if expanding folders in the search results list might
not be particularly useful most of the time, it makes sense to make the
model more robust to prevent crashes and other unexpected behavior in
such situations.

This patch makes the following changes to achieve that goal:

* Change the argument of removeItems() from KFileItemList to
  KItemRangeList. To make this work, the "look the indexes up in
  m_items" code is moved from that function to slotItemsDeleted(). In
  the other places where removeItems() is called, the indexes are
  calculated directly (which is not more difficult than determining the
  removed items as a KFileItemList, if one considers that we needed the
  function childItems(KFileItem) for that, which is not needed any more
  with this patch).

* Also removeFilteredChildren() takes a KItemRangeList now. Rather than
  putting the parent KFileItems into a QSet for O(1) lookup (which
  prevents O(N^2) worst case behavior for the entire function), it uses
  a QSet<ItemData*> now, which should even be more efficient (hashing a
  pointer is cheaper than hashing a KFileItem/KUrl).

BUG: 324371
BUG: 325359
FIXED-IN: 4.12.0
REVIEW: 113070

dolphin/src/kitemviews/kfileitemmodel.cpp

@@ -444,11 +444,18 @@ bool KFileItemModel::setExpanded(int index, bool expanded)
         m_expandedDirs.remove(targetUrl);
         m_dirLister->stop(url);
 
-        removeFilteredChildren(KFileItemList() << item);
+        const int parentLevel = expandedParentsCount(index);
+        const int itemCount = m_itemData.count();
+        const int firstChildIndex = index + 1;
 
-        const KFileItemList itemsToRemove = childItems(item);
+        int childIndex = firstChildIndex;
-        removeFilteredChildren(itemsToRemove);
+        while (childIndex < itemCount && expandedParentsCount(childIndex) > parentLevel) {
-        removeItems(itemsToRemove, DeleteItemData);
+            ++childIndex;
+        }
+        const int childrenCount = childIndex - firstChildIndex;
+
+        removeFilteredChildren(KItemRangeList() << KItemRange(index, 1 + childrenCount));
+        removeItems(KItemRangeList() << KItemRange(firstChildIndex, childrenCount), DeleteItemData);
     }
 
     return true;
@@ -551,21 +558,25 @@ void KFileItemModel::applyFilters()
 {
     // Check which shown items from m_itemData must get
     // hidden and hence moved to m_filteredItems.
-    KFileItemList newFilteredItems;
+    QVector<int> newFilteredIndexes;
+
+    const int itemCount = m_itemData.count();
+    for (int index = 0; index < itemCount; ++index) {
+        ItemData* itemData = m_itemData.at(index);
 
-    foreach (ItemData* itemData, m_itemData) {
         // Only filter non-expanded items as child items may never
         // exist without a parent item
         if (!itemData->values.value("isExpanded").toBool()) {
             const KFileItem item = itemData->item;
             if (!m_filter.matches(item)) {
-                newFilteredItems.append(item);
+                newFilteredIndexes.append(index);
                 m_filteredItems.insert(item, itemData);
             }
         }
     }
 
-    removeItems(newFilteredItems, KeepItemData);
+    const KItemRangeList removedRanges = KItemRangeList::fromSortedContainer(newFilteredIndexes);
+    removeItems(removedRanges, KeepItemData);
 
     // Check which hidden items from m_filteredItems should
     // get visible again and hence removed from m_filteredItems.
@@ -584,22 +595,24 @@ void KFileItemModel::applyFilters()
     insertItems(newVisibleItems);
 }
 
-void KFileItemModel::removeFilteredChildren(const KFileItemList& parentsList)
+void KFileItemModel::removeFilteredChildren(const KItemRangeList& itemRanges)
 {
-    if (m_filteredItems.isEmpty()) {
+    if (m_filteredItems.isEmpty() || !m_requestRole[ExpandedParentsCountRole]) {
+        // There are either no filtered items, or it is not possible to expand
+        // folders -> there cannot be any filtered children.
         return;
     }
 
-    // First, we put the parent items into a set to provide fast lookup
+    QSet<ItemData*> parents;
-    // while iterating over m_filteredItems and prevent quadratic
+    foreach (const KItemRange& range, itemRanges) {
-    // complexity if there are N parents and N filtered items.
+        for (int index = range.index; index < range.index + range.count; ++index) {
-    const QSet<KFileItem> parents = parentsList.toSet();
+            parents.insert(m_itemData.at(index));
+        }
+    }
 
     QHash<KFileItem, ItemData*>::iterator it = m_filteredItems.begin();
     while (it != m_filteredItems.end()) {
-        const ItemData* parent = it.value()->parent;
+        if (parents.contains(it.value()->parent)) {
-
-        if (parent && parents.contains(parent->item)) {
             delete it.value();
             it = m_filteredItems.erase(it);
         } else {
@@ -849,29 +862,49 @@ void KFileItemModel::slotItemsDeleted(const KFileItemList& items)
 {
     dispatchPendingItemsToInsert();
 
-    KFileItemList itemsToRemove = items;
+    QVector<int> indexesToRemove;
-    if (m_requestRole[ExpandedParentsCountRole]) {
+    indexesToRemove.reserve(items.count());
-        // Assure that removing a parent item also results in removing all children
-        foreach (const KFileItem& item, items) {
-            itemsToRemove.append(childItems(item));
-        }
-    }
 
-    if (!m_filteredItems.isEmpty()) {
+    foreach (const KFileItem& item, items) {
-        foreach (const KFileItem& item, itemsToRemove) {
+        const KUrl url = item.url();
+        const int index = m_items.value(url, -1);
+        if (index >= 0) {
+            indexesToRemove.append(index);
+        } else {
+            // Probably the item has been filtered.
             QHash<KFileItem, ItemData*>::iterator it = m_filteredItems.find(item);
             if (it != m_filteredItems.end()) {
                 delete it.value();
                 m_filteredItems.erase(it);
             }
         }
-
-        if (m_requestRole[ExpandedParentsCountRole]) {
-            removeFilteredChildren(itemsToRemove);
-        }
     }
 
-    removeItems(itemsToRemove, DeleteItemData);
+    std::sort(indexesToRemove.begin(), indexesToRemove.end());
+
+    if (m_requestRole[ExpandedParentsCountRole] && !m_expandedDirs.isEmpty()) {
+        // Assure that removing a parent item also results in removing all children
+        QVector<int> indexesToRemoveWithChildren;
+        indexesToRemoveWithChildren.reserve(m_items.count());
+
+        const int itemCount = m_itemData.count();
+        foreach (int index, indexesToRemove) {
+            indexesToRemoveWithChildren.append(index);
+
+            const int parentLevel = expandedParentsCount(index);
+            int childIndex = index + 1;
+            while (childIndex < itemCount && expandedParentsCount(childIndex) > parentLevel) {
+                indexesToRemoveWithChildren.append(childIndex);
+                ++childIndex;
+            }
+        }
+
+        indexesToRemove = indexesToRemoveWithChildren;
+    }
+
+    const KItemRangeList itemRanges = KItemRangeList::fromSortedContainer(indexesToRemove);
+    removeFilteredChildren(itemRanges);
+    removeItems(itemRanges, DeleteItemData);
 }
 
 void KFileItemModel::slotRefreshItems(const QList<QPair<KFileItem, KFileItem> >& items)
@@ -1061,23 +1094,21 @@ void KFileItemModel::insertItems(QList<ItemData*>& newItems)
 #endif
 }
 
-void KFileItemModel::removeItems(const KFileItemList& items, RemoveItemsBehavior behavior)
+void KFileItemModel::removeItems(const KItemRangeList& itemRanges, RemoveItemsBehavior behavior)
 {
-#ifdef KFILEITEMMODEL_DEBUG
+    if (itemRanges.isEmpty()) {
-    kDebug() << "Removing " << items.count() << "items";
+        return;
-#endif
+    }
 
     m_groups.clear();
 
-    // Step 1: Determine the indexes of the removed items, remove them from
+    // Step 1: Remove the items from the hash m_items, and free the ItemData.
-    //         the hash m_items, and free the ItemData.
+    int removedItemsCount = 0;
-    QList<int> indexesToRemove;
+    foreach (const KItemRange& range, itemRanges) {
-    indexesToRemove.reserve(items.count());
+        removedItemsCount += range.count;
-    foreach (const KFileItem& item, items) {
+
-        const KUrl url = item.url();
+        for (int index = range.index; index < range.index + range.count; ++index) {
-        const int index = m_items.value(url, -1);
+            const KUrl url = m_itemData.at(index)->item.url();
-        if (index >= 0) {
-            indexesToRemove.append(index);
 
             // Prevent repeated expensive rehashing by using QHash::erase(),
             // rather than QHash::remove().
@@ -1092,14 +1123,7 @@ void KFileItemModel::removeItems(const KFileItemList& items, RemoveItemsBehavior
         }
     }
 
-    if (indexesToRemove.isEmpty()) {
-        return;
-    }
-
-    std::sort(indexesToRemove.begin(), indexesToRemove.end());
-
     // Step 2: Remove the ItemData pointers from the list m_itemData.
-    const KItemRangeList itemRanges = KItemRangeList::fromSortedContainer(indexesToRemove);
     int target = itemRanges.at(0).index;
     int source = itemRanges.at(0).index + itemRanges.at(0).count;
     int nextRange = 1;
@@ -1117,7 +1141,7 @@ void KFileItemModel::removeItems(const KFileItemList& items, RemoveItemsBehavior
         }
     }
 
-    m_itemData.erase(m_itemData.end() - indexesToRemove.count(), m_itemData.end());
+    m_itemData.erase(m_itemData.end() - removedItemsCount, m_itemData.end());
 
     // Step 3: Adjust indexes in the hash m_items, starting from the
     //         index of the first removed item.
@@ -1205,17 +1229,17 @@ int KFileItemModel::expandedParentsCount(const ItemData* data)
 
 void KFileItemModel::removeExpandedItems()
 {
-    KFileItemList expandedItems;
+    QVector<int> indexesToRemove;
 
     const int maxIndex = m_itemData.count() - 1;
     for (int i = 0; i <= maxIndex; ++i) {
         const ItemData* itemData = m_itemData.at(i);
         if (itemData->parent) {
-            expandedItems.append(itemData->item);
+            indexesToRemove.append(i);
         }
     }
 
-    removeItems(expandedItems, DeleteItemData);
+    removeItems(KItemRangeList::fromSortedContainer(indexesToRemove), DeleteItemData);
     m_expandedDirs.clear();
 
     // Also remove all filtered items which have a parent.
@@ -1963,23 +1987,6 @@ QList<QPair<int, QVariant> > KFileItemModel::genericStringRoleGroups(const QByte
     return groups;
 }
 
-KFileItemList KFileItemModel::childItems(const KFileItem& item) const
-{
-    KFileItemList items;
-
-    int index = m_items.value(item.url(), -1);
-    if (index >= 0) {
-        const int parentLevel = expandedParentsCount(index);
-        ++index;
-        while (index < m_itemData.count() && expandedParentsCount(index) > parentLevel) {
-            items.append(m_itemData.at(index)->item);
-            ++index;
-        }
-    }
-
-    return items;
-}
-
 void KFileItemModel::emitSortProgress(int resolvedCount)
 {
     // Be tolerant against a resolvedCount with a wrong range.

dolphin/src/kitemviews/kfileitemmodel.h

@@ -309,7 +309,7 @@ private:
     };
 
     void insertItems(QList<ItemData*>& items);
-    void removeItems(const KFileItemList& items, RemoveItemsBehavior behavior);
+    void removeItems(const KItemRangeList& itemRanges, RemoveItemsBehavior behavior);
 
     /**
      * Helper method for insertItems() and removeItems(): Creates
@@ -389,11 +389,6 @@ private:
      */
     bool isChildItem(int index) const;
 
-    /**
-     * @return Recursive list of child items that have \a item as upper most parent.
-     */
-    KFileItemList childItems(const KFileItem& item) const;
-
     /**
      * Is invoked by KFileItemModelRolesUpdater and results in emitting the
      * sortProgress signal with a percent-value of the progress.
@@ -409,7 +404,7 @@ private:
      * Removes filtered items whose expanded parents have been deleted
      * or collapsed via setExpanded(parentIndex, false).
      */
-    void removeFilteredChildren(const KFileItemList& parentsList);
+    void removeFilteredChildren(const KItemRangeList& parents);
 
     /**
      * Maps the QByteArray-roles to RoleTypes and provides translation- and

dolphin/src/tests/kfileitemmodelbenchmark.cpp

@@ -185,7 +185,7 @@ void KFileItemModelBenchmark::insertAndRemoveManyItems()
         QCOMPARE(model.count(), initialItems.count() + newItems.count());
 
         if (!removedItems.isEmpty()) {
-            model.removeItems(removedItems, KFileItemModel::DeleteItemData);
+            model.slotItemsDeleted(removedItems);
         }
         QCOMPARE(model.count(), initialItems.count() + newItems.count() - removedItems.count());
     }

dolphin/src/tests/kfileitemmodeltest.cpp

@@ -89,6 +89,7 @@ private slots:
     void testGeneralParentChildRelationships();
     void testNameRoleGroups();
     void testNameRoleGroupsWithExpandedItems();
+    void testInconsistentModel();
 
 private:
     QStringList itemsInModel() const;
@@ -1404,6 +1405,63 @@ void KFileItemModelTest::testNameRoleGroupsWithExpandedItems()
     QCOMPARE(m_model->groups(), expectedGroups);
 }
 
+void KFileItemModelTest::testInconsistentModel()
+{
+    QSet<QByteArray> modelRoles = m_model->roles();
+    modelRoles << "isExpanded" << "isExpandable" << "expandedParentsCount";
+    m_model->setRoles(modelRoles);
+
+    QStringList files;
+    files << "a/b/c1.txt" << "a/b/c2.txt";
+
+    m_testDir->createFiles(files);
+
+    m_model->loadDirectory(m_testDir->url());
+    QVERIFY(QTest::kWaitForSignal(m_model, SIGNAL(itemsInserted(KItemRangeList)), DefaultTimeout));
+    QCOMPARE(itemsInModel(), QStringList() << "a");
+
+    // Expand "a/" and "a/b/".
+    m_model->setExpanded(0, true);
+    QVERIFY(m_model->isExpanded(0));
+    QVERIFY(QTest::kWaitForSignal(m_model, SIGNAL(itemsInserted(KItemRangeList)), DefaultTimeout));
+    QCOMPARE(itemsInModel(), QStringList() << "a" << "b");
+
+    m_model->setExpanded(1, true);
+    QVERIFY(m_model->isExpanded(1));
+    QVERIFY(QTest::kWaitForSignal(m_model, SIGNAL(itemsInserted(KItemRangeList)), DefaultTimeout));
+    QCOMPARE(itemsInModel(), QStringList() << "a" << "b" << "c1.txt" << "c2.txt");
+
+    // Add the files "c1.txt" and "c2.txt" to the model also as top-level items.
+    // Such a thing can in principle happen when performing a search, and there
+    // are files which
+    // (a) match the search string, and
+    // (b) are children of a folder that matches the search string and is expanded.
+    //
+    // Note that the first item in the list of added items must be new (i.e., not
+    // in the model yet). Otherwise, KFileItemModel::slotItemsAdded() will see that
+    // it receives items that are in the model already and ignore them.
+    KUrl url(m_model->directory().url() + "/a2");
+    KFileItem newItem(KFileItem::Unknown, KFileItem::Unknown, url);
+
+    KFileItemList items;
+    items << newItem << m_model->fileItem(2) << m_model->fileItem(3);
+    m_model->slotItemsAdded(m_model->directory(), items);
+    m_model->slotCompleted();
+    QCOMPARE(itemsInModel(), QStringList() << "a" << "b" << "c1.txt" << "c2.txt" << "a2" << "c1.txt" << "c2.txt");
+
+    m_model->setExpanded(0, false);
+
+    // Test that the right items have been removed, see
+    // https://bugs.kde.org/show_bug.cgi?id=324371
+    QCOMPARE(itemsInModel(), QStringList() << "a" << "a2" << "c1.txt" << "c2.txt");
+
+    // Test that resorting does not cause a crash, see
+    // https://bugs.kde.org/show_bug.cgi?id=325359
+    // The crash is not 100% reproducible, but Valgrind will report an invalid memory access.
+    m_model->resortAllItems();
+
+}
+
 QStringList KFileItemModelTest::itemsInModel() const
 {
     QStringList items;