KDirectoryContentsCounter: show intermediate dir size counting results, improve stopping, improve data caching

Two user visible changes:
 * we can see the dir size changing as it is updated.
 * This makes the file counting a lot more reactive, when changing directories for instance.

`KDirectoryContentsCounterWorker::walkDir` is not recursive anymore.

The cache is now shared and only a single thread is used to count size recursively.

src/kitemviews/kfileitemlistview.cpp

@@ -41,7 +41,6 @@ KFileItemListView::KFileItemListView(QGraphicsWidget *parent)
     , m_modelRolesUpdater(nullptr)
     , m_updateVisibleIndexRangeTimer(nullptr)
     , m_updateIconSizeTimer(nullptr)
-    , m_scanDirectories(true)
 {
     setAcceptDrops(true);
 
@@ -119,19 +118,6 @@ qlonglong KFileItemListView::localFileSizePreviewLimit() const
     return m_modelRolesUpdater ? m_modelRolesUpdater->localFileSizePreviewLimit() : 0;
 }
 
-void KFileItemListView::setScanDirectories(bool enabled)
-{
-    m_scanDirectories = enabled;
-    if (m_modelRolesUpdater) {
-        m_modelRolesUpdater->setScanDirectories(m_scanDirectories);
-    }
-}
-
-bool KFileItemListView::scanDirectories()
-{
-    return m_scanDirectories;
-}
-
 QPixmap KFileItemListView::createDragPixmap(const KItemSet &indexes) const
 {
     if (!model()) {
@@ -269,7 +255,6 @@ void KFileItemListView::onModelChanged(KItemModelBase *current, KItemModelBase *
     if (current) {
         m_modelRolesUpdater = new KFileItemModelRolesUpdater(static_cast<KFileItemModel *>(current), this);
         m_modelRolesUpdater->setIconSize(availableIconSize());
-        m_modelRolesUpdater->setScanDirectories(scanDirectories());
 
         applyRolesToModel();
     }

src/kitemviews/kfileitemlistview.h

@@ -71,13 +71,6 @@ public:
     void setLocalFileSizePreviewLimit(qlonglong size);
     qlonglong localFileSizePreviewLimit() const;
 
-    /**
-     * If set to true, directories contents are scanned to determine their size
-     * Default true
-     */
-    void setScanDirectories(bool enabled);
-    bool scanDirectories();
-
     QPixmap createDragPixmap(const KItemSet &indexes) const override;
 
     /**
@@ -137,7 +130,6 @@ private:
     KFileItemModelRolesUpdater *m_modelRolesUpdater;
     QTimer *m_updateVisibleIndexRangeTimer;
     QTimer *m_updateIconSizeTimer;
-    bool m_scanDirectories;
 
     friend class KFileItemListViewTest; // For unit testing
 };

src/kitemviews/kfileitemlistwidget.cpp

@@ -67,7 +67,7 @@ QString KFileItemListWidgetInformant::roleText(const QByteArray &role, const QHa
         if (values.value("isDir").toBool()) {
             if (!roleValue.isNull() && roleValue != -1) {
                 // The item represents a directory.
-                if (DetailsModeSettings::directorySizeCount()) {
+                if (DetailsModeSettings::directorySizeCount() || roleValue == -2 /* size is invalid */) {
                     //  Show the number of sub directories instead of the file size of the directory.
                     const int count = values.value("count").toInt();
                     text = i18ncp("@item:intable", "%1 item", "%1 items", count);

src/kitemviews/kfileitemmodel.cpp

@@ -129,6 +129,8 @@ void KFileItemModel::refreshDirectory(const QUrl &url)
     }
 
     m_dirLister->openUrl(url, KDirLister::Reload);
+
+    Q_EMIT directoryRefreshing();
 }
 
 QUrl KFileItemModel::directory() const

src/kitemviews/kfileitemmodel.h

@@ -217,6 +217,11 @@ Q_SIGNALS:
      */
     void directoryLoadingCompleted();
 
+    /**
+     * Is emitted when the model is being refreshed (F5 key press)
+     */
+    void directoryRefreshing();
+
     /**
      * Is emitted after the loading of a directory has been canceled.
      */

src/kitemviews/kfileitemmodelrolesupdater.cpp

@@ -10,6 +10,7 @@
 #include "kfileitemmodel.h"
 #include "private/kdirectorycontentscounter.h"
 #include "private/kpixmapmodifier.h"
+#include "qdir.h"
 
 #include <KConfig>
 #include <KConfigGroup>
@@ -20,6 +21,8 @@
 #include <KPluginMetaData>
 #include <KSharedConfig>
 
+#include "dolphin_detailsmodesettings.h"
+
 #if HAVE_BALOO
 #include "private/kbaloorolesprovider.h"
 #include <Baloo/File>
@@ -72,7 +75,6 @@ KFileItemModelRolesUpdater::KFileItemModelRolesUpdater(KFileItemModel *model, QO
     , m_resolvableRoles()
     , m_enabledPlugins()
     , m_localFileSizePreviewLimit(0)
-    , m_scanDirectories(true)
     , m_pendingSortRoleItems()
     , m_pendingIndexes()
     , m_pendingPreviewItems()
@@ -321,16 +323,6 @@ qlonglong KFileItemModelRolesUpdater::localFileSizePreviewLimit() const
     return m_localFileSizePreviewLimit;
 }
 
-void KFileItemModelRolesUpdater::setScanDirectories(bool enabled)
-{
-    m_scanDirectories = enabled;
-}
-
-bool KFileItemModelRolesUpdater::scanDirectories() const
-{
-    return m_scanDirectories;
-}
-
 void KFileItemModelRolesUpdater::setHoverSequenceState(const QUrl &itemUrl, int seqIdx)
 {
     const KFileItem item = m_model->fileItem(itemUrl);
@@ -423,8 +415,7 @@ void KFileItemModelRolesUpdater::slotItemsRemoved(const KItemRangeList &itemRang
         m_hoverSequenceLoadedItems.clear();
 
         killPreviewJob();
-
+        if (!m_model->showDirectoriesOnly()) {
-        if (m_scanDirectories) {
             m_directoryContentsCounter->stopWorker();
         }
     } else {
@@ -856,10 +847,10 @@ void KFileItemModelRolesUpdater::applyChangedBalooRolesForItem(const KFileItem &
 #endif
 }
 
-void KFileItemModelRolesUpdater::slotDirectoryContentsCountReceived(const QString &path, int count, long size)
+void KFileItemModelRolesUpdater::slotDirectoryContentsCountReceived(const QString &path, int count, long long size)
 {
-    const bool getSizeRole = m_roles.contains("size");
     const bool getIsExpandableRole = m_roles.contains("isExpandable");
+    const bool getSizeRole = m_roles.contains("size");
 
     if (getSizeRole || getIsExpandableRole) {
         const int index = m_model->index(QUrl::fromLocalFile(path));
@@ -1278,18 +1269,71 @@ bool KFileItemModelRolesUpdater::applyResolvedRoles(int index, ResolveHint hint)
 
 void KFileItemModelRolesUpdater::startDirectorySizeCounting(const KFileItem &item, int index)
 {
-    if (item.isSlow()) {
+    if (DetailsModeSettings::directorySizeCount() || item.isSlow() || !item.isLocalFile()) {
+        // fastpath no recursion necessary
+
+        auto data = m_model->data(index);
+        if (data.value("size") == -2) {
+            // means job already started
+            return;
+        }
+
+        auto url = item.url();
+        if (!item.localPath().isEmpty()) {
+            // optimization for desktop:/, trash:/
+            url = QUrl::fromLocalFile(item.localPath());
+        }
+
+        data.insert("isExpandable", false);
+        data.insert("count", 0);
+        data.insert("size", -2); // invalid size, -1 means size unknown
+
+        disconnect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+        m_model->setData(index, data);
+        connect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+
+        auto listJob = KIO::listDir(url);
+        QObject::connect(listJob, &KIO::ListJob::entries, this, [this, index](const KJob * /*job*/, const KIO::UDSEntryList &list) {
+            auto data = m_model->data(index);
+            int origCount = data.value("count").toInt();
+            int entryCount = origCount;
+
+            for (const KIO::UDSEntry &entry : list) {
+                const auto name = entry.stringValue(KIO::UDSEntry::UDS_NAME);
+
+                if (name == QStringLiteral("..") || name == QStringLiteral(".")) {
+                    continue;
+                }
+                if (!m_model->showHiddenFiles() && name.startsWith(QLatin1Char('.'))) {
+                    continue;
+                }
+                if (m_model->showDirectoriesOnly() && !entry.isDir()) {
+                    continue;
+                }
+                ++entryCount;
+            }
+
+            // count has changed
+            if (origCount < entryCount) {
+                QHash<QByteArray, QVariant> data;
+                data.insert("isExpandable", entryCount > 0);
+                data.insert("count", entryCount);
+
+                disconnect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+                m_model->setData(index, data);
+                connect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+            }
+        });
         return;
     }
 
     // Tell m_directoryContentsCounter that we want to count the items
     // inside the directory. The result will be received in slotDirectoryContentsCountReceived.
-    if (m_scanDirectories && item.isLocalFile()) {
+    const QString path = item.localPath();
-        const QString path = item.localPath();
+    const auto priority = index >= m_firstVisibleIndex && index <= m_lastVisibleIndex ? KDirectoryContentsCounter::PathCountPriority::High
-        const auto priority = index >= m_firstVisibleIndex && index <= m_lastVisibleIndex ? KDirectoryContentsCounter::PathCountPriority::High
+                                                                                      : KDirectoryContentsCounter::PathCountPriority::Normal;
-                                                                                          : KDirectoryContentsCounter::PathCountPriority::Normal;
+
-        m_directoryContentsCounter->scanDirectory(path, priority);
+    m_directoryContentsCounter->scanDirectory(path, priority);
-    }
 }
 
 QHash<QByteArray, QVariant> KFileItemModelRolesUpdater::rolesData(const KFileItem &item, int index)

src/kitemviews/kfileitemmodelrolesupdater.h

@@ -260,7 +260,7 @@ private Q_SLOTS:
     void applyChangedBalooRoles(const QString &file);
     void applyChangedBalooRolesForItem(const KFileItem &file);
 
-    void slotDirectoryContentsCountReceived(const QString &path, int count, long size);
+    void slotDirectoryContentsCountReceived(const QString &path, int count, long long size);
 
 private:
     /**
@@ -334,6 +334,9 @@ private:
     void trimHoverSequenceLoadedItems();
 
 private:
+    /**
+     * enqueue directory size counting for KFileItem item at index
+     */
     void startDirectorySizeCounting(const KFileItem &item, int index);
 
     enum State { Idle, Paused, ResolvingSortRole, ResolvingAllRoles, PreviewJobRunning };
@@ -370,7 +373,6 @@ private:
     QSet<QByteArray> m_resolvableRoles;
     QStringList m_enabledPlugins;
     qulonglong m_localFileSizePreviewLimit;
-    bool m_scanDirectories;
 
     // Items for which the sort role still has to be determined.
     QSet<KFileItem> m_pendingSortRoleItems;

src/kitemviews/private/kdirectorycontentscounter.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "kdirectorycontentscounter.h"
+#include "dolphin_detailsmodesettings.h"
 #include "kitemviews/kfileitemmodel.h"
 
 #include <KDirWatch>
@@ -16,36 +17,102 @@
 
 namespace
 {
+
+class LocalCache
+{
+public:
+    struct cacheData {
+        int count = 0;
+        long long size = 0;
+        ushort refCount = 0;
+        qint64 timestamp = 0;
+
+        inline operator bool() const
+        {
+            return timestamp != 0 && count != -1;
+        }
+    };
+
+    LocalCache()
+        : m_cache()
+    {
+    }
+
+    cacheData insert(const QString &key, cacheData data, bool inserted)
+    {
+        data.timestamp = QDateTime::currentMSecsSinceEpoch();
+        if (inserted) {
+            data.refCount += 1;
+        }
+        m_cache.insert(key, data);
+        return data;
+    }
+
+    cacheData value(const QString &key) const
+    {
+        return m_cache.value(key);
+    }
+
+    void unRefAll(const QSet<QString> &keys)
+    {
+        for (const auto &key : keys) {
+            auto entry = m_cache[key];
+            entry.refCount -= 1;
+            if (entry.refCount == 0) {
+                m_cache.remove(key);
+            }
+        }
+    }
+
+    void removeAll(const QSet<QString> &keys)
+    {
+        for (const auto &key : keys) {
+            m_cache.remove(key);
+        }
+    }
+
+private:
+    QHash<QString, cacheData> m_cache;
+};
+
 /// cache of directory counting result
-static QHash<QString, QPair<int, long>> *s_cache;
+static LocalCache *s_cache;
+static QThread *s_workerThread;
+static KDirectoryContentsCounterWorker *s_worker;
 }
 
 KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObject *parent)
     : QObject(parent)
     , m_model(model)
+    , m_priorityQueue()
     , m_queue()
-    , m_worker(nullptr)
     , m_workerIsBusy(false)
     , m_dirWatcher(nullptr)
     , m_watchedDirs()
 {
-    connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved);
-
-    if (!m_workerThread) {
-        m_workerThread = new QThread();
-        m_workerThread->start();
-    }
-
     if (s_cache == nullptr) {
-        s_cache = new QHash<QString, QPair<int, long>>();
+        s_cache = new LocalCache();
     }
 
-    m_worker = new KDirectoryContentsCounterWorker();
+    if (!s_workerThread) {
-    m_worker->moveToThread(m_workerThread);
+        s_workerThread = new QThread();
+        s_workerThread->setObjectName(QStringLiteral("KDirectoryContentsCounterThread"));
+        s_workerThread->start();
+    }
 
-    connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, m_worker, &KDirectoryContentsCounterWorker::countDirectoryContents);
+    if (!s_worker) {
-    connect(this, &KDirectoryContentsCounter::stop, m_worker, &KDirectoryContentsCounterWorker::stop);
+        s_worker = new KDirectoryContentsCounterWorker();
-    connect(m_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult);
+        s_worker->moveToThread(s_workerThread);
+    }
+
+    connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved);
+    connect(m_model, &KFileItemModel::directoryRefreshing, this, &KDirectoryContentsCounter::slotDirectoryRefreshing);
+
+    connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, s_worker, &KDirectoryContentsCounterWorker::countDirectoryContents);
+
+    connect(s_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult);
+    connect(s_worker, &KDirectoryContentsCounterWorker::intermediateResult, this, &KDirectoryContentsCounter::result);
+    connect(s_worker, &KDirectoryContentsCounterWorker::finished, this, &KDirectoryContentsCounter::scheduleNext);
 
     m_dirWatcher = new KDirWatch(this);
     connect(m_dirWatcher, &KDirWatch::dirty, this, &KDirectoryContentsCounter::slotDirWatchDirty);
@@ -53,60 +120,20 @@ KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObj
 
 KDirectoryContentsCounter::~KDirectoryContentsCounter()
 {
-    if (m_workerThread->isRunning()) {
+    s_cache->unRefAll(m_watchedDirs);
-        // The worker thread will continue running. It could even be running
-        // a method of m_worker at the moment, so we delete it using
-        // deleteLater() to prevent a crash.
-        m_worker->deleteLater();
-    } else {
-        // There are no remaining workers -> stop the worker thread.
-        m_workerThread->quit();
-        m_workerThread->wait();
-        delete m_workerThread;
-        m_workerThread = nullptr;
-
-        // The worker thread has finished running now, so it's safe to delete
-        // m_worker. deleteLater() would not work at all because the event loop
-        // which would deliver the event to m_worker is not running any more.
-        delete m_worker;
-    }
 }
 
-void KDirectoryContentsCounter::slotResult(const QString &path, int count, long size)
+void KDirectoryContentsCounter::slotResult(const QString &path, int count, long long size)
 {
-    m_workerIsBusy = false;
+    const auto fileInfo = QFileInfo(path);
-
+    const QString resolvedPath = fileInfo.canonicalFilePath();
-    const QFileInfo info = QFileInfo(path);
+    if (fileInfo.isReadable() && !m_watchedDirs.contains(resolvedPath)) {
-    const QString resolvedPath = info.canonicalFilePath();
-
-    if (!m_dirWatcher->contains(resolvedPath)) {
         m_dirWatcher->addDir(resolvedPath);
-        m_watchedDirs.insert(resolvedPath);
     }
+    bool inserted = m_watchedDirs.insert(resolvedPath) == m_watchedDirs.end();
 
-    if (!m_priorityQueue.empty()) {
+    // update cache or overwrite value
-        const QString firstPath = m_priorityQueue.front();
+    s_cache->insert(resolvedPath, {count, size, true}, inserted);
-        m_priorityQueue.pop_front();
-        scanDirectory(firstPath, PathCountPriority::High);
-    } else if (!m_queue.empty()) {
-        const QString firstPath = m_queue.front();
-        m_queue.pop_front();
-        scanDirectory(firstPath, PathCountPriority::Normal);
-    }
-
-    if (s_cache->contains(resolvedPath)) {
-        const auto pair = s_cache->value(resolvedPath);
-        if (pair.first == count && pair.second == size) {
-            // no change no need to send another result event
-            return;
-        }
-    }
-
-    if (info.dir().path() == m_model->rootItem().url().path()) {
-        // update cache or overwrite value
-        // when path is a direct children of the current model root
-        s_cache->insert(resolvedPath, QPair<int, long>(count, size));
-    }
 
     // sends the results
     Q_EMIT result(path, count, size);
@@ -131,6 +158,11 @@ void KDirectoryContentsCounter::slotItemsRemoved()
 {
     const bool allItemsRemoved = (m_model->count() == 0);
 
+    if (allItemsRemoved) {
+        s_cache->removeAll(m_watchedDirs);
+        stopWorker();
+    }
+
     if (!m_watchedDirs.isEmpty()) {
         // Don't let KDirWatch watch for removed items
         if (allItemsRemoved) {
@@ -138,7 +170,6 @@ void KDirectoryContentsCounter::slotItemsRemoved()
                 m_dirWatcher->removeDir(path);
             }
             m_watchedDirs.clear();
-            m_queue.clear();
         } else {
             QMutableSetIterator<QString> it(m_watchedDirs);
             while (it.hasNext()) {
@@ -152,46 +183,102 @@ void KDirectoryContentsCounter::slotItemsRemoved()
     }
 }
 
-void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority)
+void KDirectoryContentsCounter::slotDirectoryRefreshing()
 {
-    const QString resolvedPath = QFileInfo(path).canonicalFilePath();
+    s_cache->removeAll(m_watchedDirs);
-    const bool alreadyInCache = s_cache->contains(resolvedPath);
+}
-    if (alreadyInCache) {
+
-        // fast path when in cache
+void KDirectoryContentsCounter::scheduleNext()
-        // will be updated later if result has changed
+{
-        const auto pair = s_cache->value(resolvedPath);
+    if (!m_priorityQueue.empty()) {
-        Q_EMIT result(path, pair.first, pair.second);
+        m_currentPath = m_priorityQueue.front();
+        m_priorityQueue.pop_front();
+    } else if (!m_queue.empty()) {
+        m_currentPath = m_queue.front();
+        m_queue.pop_front();
+    } else {
+        m_currentPath.clear();
+        m_workerIsBusy = false;
+        return;
     }
 
-    if (m_workerIsBusy) {
+    const auto fileInfo = QFileInfo(m_currentPath);
-        // only enqueue path not yet in queue
+    const QString resolvedPath = fileInfo.canonicalFilePath();
-        if (std::find(m_queue.begin(), m_queue.end(), path) == m_queue.end()
+    const auto pair = s_cache->value(resolvedPath);
-            && std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path) == m_priorityQueue.end()) {
+    if (pair) {
-            if (priority == PathCountPriority::Normal) {
+        // fast path when in cache
-                if (alreadyInCache) {
+        // will be updated later if result has changed
-                    // if we already knew the dir size, it gets lower priority
+        Q_EMIT result(m_currentPath, pair.count, pair.size);
-                    m_queue.push_back(path);
+    }
-                } else {
+
-                    m_queue.push_front(path);
+    // if scanned fully recently, skip rescan
-                }
+    if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) {
-            } else {
+        scheduleNext();
-                // append to priority queue
+        return;
-                m_priorityQueue.push_back(path);
+    }
-            }
+
+    KDirectoryContentsCounterWorker::Options options;
+
+    if (m_model->showHiddenFiles()) {
+        options |= KDirectoryContentsCounterWorker::CountHiddenFiles;
+    }
+
+    m_workerIsBusy = true;
+    Q_EMIT requestDirectoryContentsCount(m_currentPath, options, DetailsModeSettings::recursiveDirectorySizeLimit());
+}
+
+void KDirectoryContentsCounter::enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority)
+{
+    // ensure to update the entry in the queue
+    auto it = std::find(m_queue.begin(), m_queue.end(), path);
+    if (it != m_queue.end()) {
+        m_queue.erase(it);
+    } else {
+        it = std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path);
+        if (it != m_priorityQueue.end()) {
+            m_priorityQueue.erase(it);
+        }
+    }
+
+    if (priority == PathCountPriority::Normal) {
+        if (alreadyInCache) {
+            // we already knew the dir size
+            // otherwise it gets lower priority
+            m_queue.push_back(path);
+        } else {
+            m_queue.push_front(path);
         }
     } else {
-        KDirectoryContentsCounterWorker::Options options;
+        // append to priority queue
+        m_priorityQueue.push_front(path);
+    }
+}
 
-        if (m_model->showHiddenFiles()) {
+void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority)
-            options |= KDirectoryContentsCounterWorker::CountHiddenFiles;
+{
-        }
+    if (m_workerIsBusy && m_currentPath == path) {
+        // already listing
+        return;
+    }
 
-        if (m_model->showDirectoriesOnly()) {
+    const auto fileInfo = QFileInfo(path);
-            options |= KDirectoryContentsCounterWorker::CountDirectoriesOnly;
+    const QString resolvedPath = fileInfo.canonicalFilePath();
-        }
+    const auto pair = s_cache->value(resolvedPath);
+    if (pair) {
+        // fast path when in cache
+        // will be updated later if result has changed
+        Q_EMIT result(path, pair.count, pair.size);
+    }
 
-        Q_EMIT requestDirectoryContentsCount(path, options);
+    // if scanned fully recently, skip rescan
-        m_workerIsBusy = true;
+    if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) {
+        return;
+    }
+
+    enqueuePathScanning(path, pair, priority);
+
+    if (!m_workerIsBusy && !s_worker->stopping()) {
+        scheduleNext();
     }
 }
 
@@ -199,7 +286,9 @@ void KDirectoryContentsCounter::stopWorker()
 {
     m_queue.clear();
     m_priorityQueue.clear();
-    Q_EMIT stop();
-}
 
-QThread *KDirectoryContentsCounter::m_workerThread = nullptr;
+    if (m_workerIsBusy && m_currentPath == s_worker->scannedPath()) {
+        s_worker->stop();
+    }
+    m_currentPath.clear();
+}

src/kitemviews/private/kdirectorycontentscounter.h

@@ -49,32 +49,32 @@ Q_SIGNALS:
      * Signals that the directory \a path contains \a count items of size \a
      * Size calculation depends on parameter DetailsModeSettings::recursiveDirectorySizeLimit
      */
-    void result(const QString &path, int count, long size);
+    void result(const QString &path, int count, long long size);
 
-    void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options);
+    void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel);
-
-    void stop();
 
 private Q_SLOTS:
-    void slotResult(const QString &path, int count, long size);
+    void slotResult(const QString &path, int count, long long size);
     void slotDirWatchDirty(const QString &path);
     void slotItemsRemoved();
+    void slotDirectoryRefreshing();
+    void scheduleNext();
 
 private:
+    void enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority);
+
     KFileItemModel *m_model;
 
     // Used as FIFO queues.
     std::list<QString> m_priorityQueue;
     std::list<QString> m_queue;
 
-    static QThread *m_workerThread;
-
-    KDirectoryContentsCounterWorker *m_worker;
     bool m_workerIsBusy;
 
     KDirWatch *m_dirWatcher;
     QSet<QString> m_watchedDirs; // Required as sadly KDirWatch does not offer a getter method
                                  // to get all watched directories.
+    QString m_currentPath;
 };
 
 #endif

src/kitemviews/private/kdirectorycontentscounterworker.cpp

@@ -7,16 +7,16 @@
 
 #include "kdirectorycontentscounterworker.h"
 
-// Required includes for subItemsCount():
+// Required includes for countDirectoryContents():
 #ifdef Q_OS_WIN
 #include <QDir>
 #else
-#include <QFile>
+#include <QElapsedTimer>
-#include <qplatformdefs.h>
+#include <fts.h>
+#include <sys/stat.h>
+#include <sys/types.h>
 #endif
 
-#include "dolphin_detailsmodesettings.h"
-
 KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent)
     : QObject(parent)
 {
@@ -24,100 +24,135 @@ KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent
 }
 
 #ifndef Q_OS_WIN
-KDirectoryContentsCounterWorker::CountResult
+void KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel)
-KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel)
 {
-    int count = -1;
+    QByteArray text = dirPath.toLocal8Bit();
-    long size = -1;
+    char *rootPath = new char[text.size() + 1];
-    auto dir = QT_OPENDIR(QFile::encodeName(dirPath));
+    ::strncpy(rootPath, text.constData(), text.size() + 1);
-    if (dir) {
+    char *path[2]{rootPath, nullptr};
-        count = 0;
-        size = 0;
-        QT_DIRENT *dirEntry;
-        QT_STATBUF buf;
 
-        while (!m_stopping && (dirEntry = QT_READDIR(dir))) {
+    // follow symlink only for root dir
-            if (dirEntry->d_name[0] == '.') {
+    auto tree = ::fts_open(path, FTS_COMFOLLOW | FTS_PHYSICAL | FTS_XDEV, nullptr);
-                if (dirEntry->d_name[1] == '\0' || !countHiddenFiles) {
+    if (!tree) {
-                    // Skip "." or hidden files
+        delete[] rootPath;
-                    continue;
+        return;
-                }
+    }
-                if (dirEntry->d_name[1] == '.' && dirEntry->d_name[2] == '\0') {
+
-                    // Skip ".."
+    FTSENT *node;
-                    continue;
+    long long totalSize = -1;
-                }
+    int totalCount = -1;
+    QElapsedTimer timer;
+    timer.start();
+
+    while ((node = fts_read(tree)) && !m_stopping) {
+        auto info = node->fts_info;
+
+        if (info == FTS_DC) {
+            // ignore directories clausing cycles
+            continue;
+        }
+        if (info == FTS_DNR) {
+            // ignore directories that can’t be read
+            continue;
+        }
+        if (info == FTS_ERR) {
+            // ignore directories causing errors
+            fts_set(tree, node, FTS_SKIP);
+            continue;
+        }
+        if (info == FTS_DP) {
+            // ignore end traversal of dir
+            continue;
+        }
+
+        if (!countHiddenFiles && node->fts_name[0] == '.' && strncmp(".git", node->fts_name, 4) != 0) {
+            // skip hidden files, except .git dirs
+            if (info == FTS_D) {
+                fts_set(tree, node, FTS_SKIP);
             }
+            continue;
+        }
 
-            // If only directories are counted, consider an unknown file type and links also
+        if (info == FTS_F) {
-            // as directory instead of trying to do an expensive stat()
+            // only count files that are physical (aka skip /proc/kcore...)
-            // (see bugs 292642 and 299997).
+            // naive size counting not taking into account effective disk space used (aka size/block_size * block_size)
-            const bool countEntry = !countDirectoriesOnly || dirEntry->d_type == DT_DIR || dirEntry->d_type == DT_LNK || dirEntry->d_type == DT_UNKNOWN;
+            // skip directory size (usually a 4KB block)
-            if (countEntry) {
+            if (node->fts_statp->st_blocks > 0) {
-                ++count;
+                totalSize += node->fts_statp->st_size;
-            }
-
-            if (allowedRecursiveLevel > 0) {
-                QString nameBuf = QStringLiteral("%1/%2").arg(dirPath, dirEntry->d_name);
-
-                if (dirEntry->d_type == DT_REG) {
-                    if (QT_STAT(nameBuf.toLocal8Bit(), &buf) == 0) {
-                        size += buf.st_size;
-                    }
-                }
-                if (!m_stopping && dirEntry->d_type == DT_DIR) {
-                    // recursion for dirs
-                    auto subdirResult = walkDir(nameBuf, countHiddenFiles, countDirectoriesOnly, allowedRecursiveLevel - 1);
-                    if (subdirResult.size > 0) {
-                        size += subdirResult.size;
-                    }
-                }
             }
         }
-        QT_CLOSEDIR(dir);
+
+        if (info == FTS_D) {
+            if (node->fts_level == 0) {
+                // first read was sucessful, we can init counters
+                totalSize = 0;
+                totalCount = 0;
+            }
+
+            if (node->fts_level > (int)allowedRecursiveLevel) {
+                // skip too deep nodes
+                fts_set(tree, node, FTS_SKIP);
+                continue;
+            }
+        }
+        // count first level elements
+        if (node->fts_level == 1) {
+            ++totalCount;
+        }
+
+        // delay intermediate results
+        if (timer.hasExpired(200) || node->fts_level == 0) {
+            Q_EMIT intermediateResult(dirPath, totalCount, totalSize);
+            timer.restart();
+        }
+    }
+
+    delete[] rootPath;
+    fts_close(tree);
+    if (errno != 0) {
+        return;
+    }
+
+    if (!m_stopping) {
+        Q_EMIT result(dirPath, totalCount, totalSize);
     }
-    return KDirectoryContentsCounterWorker::CountResult{count, size};
 }
 #endif
 
-KDirectoryContentsCounterWorker::CountResult KDirectoryContentsCounterWorker::subItemsCount(const QString &path, Options options)
-{
-    const bool countHiddenFiles = options & CountHiddenFiles;
-    const bool countDirectoriesOnly = options & CountDirectoriesOnly;
-
-#ifdef Q_OS_WIN
-    QDir dir(path);
-    QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System;
-    if (countHiddenFiles) {
-        filters |= QDir::Hidden;
-    }
-    if (countDirectoriesOnly) {
-        filters |= QDir::Dirs;
-    } else {
-        filters |= QDir::AllEntries;
-    }
-    return {static_cast<int>(dir.entryList(filters).count()), 0};
-#else
-
-    const uint maxRecursiveLevel = DetailsModeSettings::directorySizeCount() ? 1 : DetailsModeSettings::recursiveDirectorySizeLimit();
-
-    auto res = walkDir(path, countHiddenFiles, countDirectoriesOnly, maxRecursiveLevel);
-
-    return res;
-#endif
-}
-
 void KDirectoryContentsCounterWorker::stop()
 {
     m_stopping = true;
 }
 
-void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options)
+bool KDirectoryContentsCounterWorker::stopping() const
 {
-    m_stopping = false;
+    return m_stopping;
-
+}
-    auto res = subItemsCount(path, options);
+
-
+QString KDirectoryContentsCounterWorker::scannedPath() const
-    if (!m_stopping) {
+{
-        Q_EMIT result(path, res.count, res.size);
+    return m_scannedPath;
-    }
+}
+
+void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options, int maxRecursiveLevel)
+{
+    const bool countHiddenFiles = options & CountHiddenFiles;
+
+#ifdef Q_OS_WIN
+    QDir dir(path);
+    QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System | QDir::AllEntries;
+    if (countHiddenFiles) {
+        filters |= QDir::Hidden;
+    }
+
+    Q_EMIT result(path, static_cast<int>(dir.entryList(filters).count()), 0);
+#else
+
+    m_scannedPath = path;
+    walkDir(path, countHiddenFiles, maxRecursiveLevel);
+
+#endif
+
+    m_stopping = false;
+    Q_EMIT finished();
 }

src/kitemviews/private/kdirectorycontentscounterworker.h

@@ -17,32 +17,20 @@ class KDirectoryContentsCounterWorker : public QObject
     Q_OBJECT
 
 public:
-    enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1, CountDirectoriesOnly = 0x2 };
+    enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1 };
     Q_DECLARE_FLAGS(Options, Option)
 
-    struct CountResult {
-        /// number of elements in the directory
-        int count;
-        /// Recursive sum of the size of the directory content files and folders
-        /// Calculation depends on DetailsModeSettings::recursiveDirectorySizeLimit
-        long size;
-    };
-
     explicit KDirectoryContentsCounterWorker(QObject *parent = nullptr);
 
-    /**
+    bool stopping() const;
-     * Counts the items inside the directory \a path using the options
+    QString scannedPath() const;
-     * \a options.
-     *
-     * @return The number of items.
-     */
-    CountResult subItemsCount(const QString &path, Options options);
-
 Q_SIGNALS:
     /**
      * Signals that the directory \a path contains \a count items and optionally the size of its content.
      */
-    void result(const QString &path, int count, long size);
+    void result(const QString &path, int count, long long size);
+    void intermediateResult(const QString &path, int count, long long size);
+    void finished();
 
 public Q_SLOTS:
     /**
@@ -52,16 +40,17 @@ public Q_SLOTS:
     // Note that the full type name KDirectoryContentsCounterWorker::Options
     // is needed here. Just using 'Options' is OK for the compiler, but
     // confuses moc.
-    void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options);
+    void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel);
     void stop();
 
 private:
 #ifndef Q_OS_WIN
-    KDirectoryContentsCounterWorker::CountResult
+    void walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel);
-    walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel);
 #endif
 
-    bool m_stopping = false;
+    QString m_scannedPath;
+
+    std::atomic<bool> m_stopping = false;
 };
 
 Q_DECLARE_METATYPE(KDirectoryContentsCounterWorker::Options)

src/panels/folders/folderspanel.cpp

@@ -131,7 +131,6 @@ void FoldersPanel::showEvent(QShowEvent *event)
         // This assures that no performance and memory overhead is given when the folders panel is not
         // used at all and stays invisible.
         KFileItemListView *view = new KFileItemListView();
-        view->setScanDirectories(false);
         view->setWidgetCreator(new KItemListWidgetCreator<FoldersItemListWidget>());
         view->setSupportsItemExpanding(true);
         // Set the opacity to 0 initially. The opacity will be increased after the loading of the initial tree