WIP: feat: support notify_file_id push notifications

src/common/syncjournaldb.cpp

@@ -1,10 +1,10 @@
 /*
  * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
  * SPDX-FileCopyrightText: 2014 ownCloud GmbH
  * SPDX-License-Identifier: LGPL-2.1-or-later
  */
 
 #include <QCryptographicHash>
 #include <QFile>
 #include <QLoggingCategory>
 #include <QStringList>
@@ -1762,6 +1762,40 @@
     return true;
 }
 
+bool SyncJournalDb::hasFileIds(const QList<qint64> &fileIds)
+{
+    QMutexLocker locker(&_mutex);
+
+    if (!checkConnect()) {
+        return false;
+    }
+
+    QStringList fileIdStrings = {};
+    for (const auto &fileId : fileIds) {
+        fileIdStrings.append(QString::number(fileId));
+    }
+
+    // quick workaround for looking up pure numeric file IDs: simply `round()` that field!
+    // this will return the file ID as e.g. 12345.0, but that's still good enough to use it
+    // with the IN operator -- e.g. (12345, 1337, 29001)
+    SqlQuery query(
+        QLatin1String{"SELECT 1 FROM metadata WHERE ROUND(fileid) IN (%1) LIMIT 1;"}
+            .arg(fileIdStrings.join(QLatin1String{", "})).toLocal8Bit(),
+        _db
+    );
+
+    if (!query.exec()) {
+        qCWarning(lcDb) << "file id query failed:" << query.error();
+        return false;
+    }
+
+    if (query.next().hasData && query.intValue(0) == 1) {
+        return true;
+    }
+
+    return false;
+}
+
 Optional<SyncJournalDb::HasHydratedDehydrated> SyncJournalDb::hasHydratedOrDehydratedFiles(const QByteArray &filename)
 {
     QMutexLocker locker(&_mutex);

src/common/syncjournaldb.h

@@ -7,7 +7,7 @@
 #ifndef SYNCJOURNALDB_H
 #define SYNCJOURNALDB_H
 
 #include <QObject>
 #include <QDateTime>
 #include <QHash>
 #include <QMutex>
@@ -76,6 +76,8 @@
     [[nodiscard]] bool updateLocalMetadata(const QString &filename,
         qint64 modtime, qint64 size, quint64 inode, const SyncJournalFileLockInfo &lockInfo);
 
+    [[nodiscard]] bool hasFileIds(const QList<qint64> &fileIds);
+
     /// Return value for hasHydratedOrDehydratedFiles()
     struct HasHydratedDehydrated
     {

src/gui/folder.cpp

@@ -1,10 +1,10 @@
 /*
  * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
  * SPDX-FileCopyrightText: 2014 ownCloud GmbH
  * SPDX-License-Identifier: GPL-2.0-or-later
  */
 
 #include "common/syncjournaldb.h"
 #include "config.h"
 
 #include "account.h"
@@ -92,6 +92,7 @@
 
     connect(_accountState.data(), &AccountState::isConnectedChanged, this, &Folder::canSyncChanged);
     connect(_engine.data(), &SyncEngine::rootEtag, this, &Folder::etagRetrievedFromSyncEngine);
+    connect(_engine.data(), &SyncEngine::rootFileIdReceived, this, &Folder::rootFileIdReceivedFromSyncEngine);
 
     connect(_engine.data(), &SyncEngine::started, this, &Folder::slotSyncStarted, Qt::QueuedConnection);
     connect(_engine.data(), &SyncEngine::finished, this, &Folder::slotSyncFinished, Qt::QueuedConnection);
@@ -397,6 +398,11 @@
     _lastEtag = etag;
 }
 
+void Folder::rootFileIdReceivedFromSyncEngine(const qint64 fileId)
+{
+    qCDebug(lcFolder).nospace() << "retrieved root fileId=" << fileId;
+    _rootFileId = fileId;
+}
 
 void Folder::showSyncResultPopup()
 {
@@ -1003,7 +1009,12 @@
     }
 }
 
+bool Folder::hasFileIds(const QList<qint64>& fileIds) const
+{
+    return fileIds.contains(_rootFileId) || journalDb()->hasFileIds(fileIds);
+}
+
 QString Folder::filePath(const QString& fileName)
 {
     const auto folderDir = QDir(_canonicalLocalPath);
 

src/gui/folder.h

@@ -1,3 +1,3 @@
 /*
  * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors
  * SPDX-FileCopyrightText: 2014 ownCloud GmbH
@@ -7,7 +7,7 @@
 #ifndef MIRALL_FOLDER_H
 #define MIRALL_FOLDER_H
 
 #include "syncresult.h"
 #include "progressdispatcher.h"
 #include "common/syncjournaldb.h"
 #include "networkjobs.h"
@@ -316,6 +316,9 @@
     void blacklistPath(const QString &path);
     void migrateBlackListPath(const QString &legacyPath);
 
+    /// whether the current folder contains any of the passed fileIds
+    [[nodiscard]] bool hasFileIds(const QList<qint64>& fileIds) const;
+
 signals:
     void syncStateChange();
     void syncStarted();
@@ -429,6 +432,8 @@
     void etagRetrieved(const QByteArray &, const QDateTime &tp);
     void etagRetrievedFromSyncEngine(const QByteArray &, const QDateTime &time);
 
+    void rootFileIdReceivedFromSyncEngine(const qint64 fileId);
+
     void slotEmitFinishedDelayed();
 
     void slotNewBigFolderDiscovered(const QString &, bool isExternal);
@@ -585,6 +590,9 @@
     QMetaObject::Connection _officeFileLockReleaseUnlockFailure;
     QMetaObject::Connection _fileLockSuccess;
     QMetaObject::Connection _fileLockFailure;
+
+    /// The remote file ID of the current folder.
+    qint64 _rootFileId = 0;
 };
 }
 

src/gui/folderman.cpp

@@ -2079,15 +2079,35 @@ void FolderMan::slotSetupPushNotifications(const Folder::Map &folderMap)
 
 void FolderMan::slotProcessFilesPushNotification(Account *account)
 {
-    qCInfo(lcFolderMan) << "Got files push notification for account" << account;
+    qCDebug(lcFolderMan) << "received notify_file push notification account=" << account->displayName();
 
     for (auto folder : std::as_const(_folderMap)) {
         // Just run on the folders that belong to this account
         if (folder->accountState()->account() != account) {
             continue;
         }
 
-        qCInfo(lcFolderMan) << "Schedule folder" << folder << "for sync";
+        qCInfo(lcFolderMan).nospace() << "scheduling sync account=" << account->displayName() << " folder=" << folder->alias() << " reason=notify_file";
+        scheduleFolder(folder);
+    }
+}
+
+void FolderMan::slotProcessFileIdsPushNotification(Account *account, const QList<qint64> &fileIds)
+{
+    qCDebug(lcFolderMan).nospace() << "received notify_file_id push notification account=" << account->displayName() << " fileIds=" << fileIds;
+
+    for (auto folder : std::as_const(_folderMap)) {
+        // Just run on the folders that belong to this account
+        if (folder->accountState()->account() != account) {
+            continue;
+        }
+
+        if (!folder->hasFileIds(fileIds)) {
+            qCDebug(lcFolderMan).nospace() << "no matching file ids, ignoring account=" << account->displayName() << " folder=" << folder->alias();
+            continue;
+        }
+
+        qCInfo(lcFolderMan).nospace() << "scheduling sync account=" << account->displayName() << " folder=" << folder->alias() << " reason=notify_file_id";
         scheduleFolder(folder);
     }
 }
@@ -2099,6 +2119,7 @@ void FolderMan::slotConnectToPushNotifications(const AccountPtr &account)
     if (pushNotificationsFilesReady(account)) {
         qCInfo(lcFolderMan) << "Push notifications ready";
         connect(pushNotifications, &PushNotifications::filesChanged, this, &FolderMan::slotProcessFilesPushNotification, Qt::UniqueConnection);
+        connect(pushNotifications, &PushNotifications::fileIdsChanged, this, &FolderMan::slotProcessFileIdsPushNotification, Qt::UniqueConnection);
     }
 }
 

src/gui/folderman.h

@@ -7,7 +7,7 @@
 #ifndef FOLDERMAN_H
 #define FOLDERMAN_H
 
 #include <QByteArray>
 #include <QObject>
 #include <QQueue>
 #include <QList>
@@ -329,6 +329,7 @@
 
     void slotSetupPushNotifications(const OCC::Folder::Map &);
     void slotProcessFilesPushNotification(OCC::Account *account);
+    void slotProcessFileIdsPushNotification(OCC::Account *account, const QList<qint64> &fileIds);
     void slotConnectToPushNotifications(const OCC::AccountPtr &account);
 
     void slotLeaveShare(const QString &localFile, const QByteArray &folderToken = {});

src/libsync/discovery.cpp

@@ -2235,6 +2235,7 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery()
     }
 
     connect(serverJob, &DiscoverySingleDirectoryJob::etag, this, &ProcessDirectoryJob::etag);
+    connect(serverJob, &DiscoverySingleDirectoryJob::rootFileIdReceived, this, &ProcessDirectoryJob::rootFileIdReceived);
     connect(serverJob, &DiscoverySingleDirectoryJob::setfolderQuota, this, &ProcessDirectoryJob::setFolderQuota);
     _discoveryData->_currentlyActiveJobs++;
     _pendingAsyncJobs++;

src/libsync/discovery.h

@@ -6,7 +6,7 @@
 
 #pragma once
 
 #include <QObject>
 #include <cstdint>
 #include "csync_exclude.h"
 #include "discoveryphase.h"
@@ -306,6 +306,7 @@
     // The root etag of this directory was fetched
     void etag(const QByteArray &, const QDateTime &time);
     void updatedRootFolderQuota(const int64_t &bytesUsed, const int64_t &bytesAvailable);
+    void rootFileIdReceived(const qint64 fileId);
 
 private slots:
     void setFolderQuota(const FolderQuota &folderQuota);

src/libsync/discoveryphase.cpp

@@ -590,7 +590,7 @@
     }
 }
 
 void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &file, const QMap<QString, QString> &map)
 {
     if (!_ignoredFirst) {
         // The first entry is for the folder itself, we should process it differently.
@@ -613,8 +613,17 @@
             _localFileId = map.value(QStringLiteral("fileid")).toUtf8();
         }
         if (map.contains("id")) {
+            // this is from the "oc:id" property, the format is e.g. "00000002oc123xyz987e"
             _fileId = map.value("id").toUtf8();
         }
+        if (map.contains("fileid")) {
+            // this is from the "oc:fileid" property, this is the plain ID without any special format (e.g. "2")
+            bool ok = false;
+            if (qint64 numericFileId = map.value("fileid").toLongLong(&ok); ok) {
+                qCDebug(lcDiscovery).nospace() << "received numericFileId=" << numericFileId;
+                emit rootFileIdReceived(numericFileId);
+            }
+        }
         if (map.contains("is-encrypted") && map.value("is-encrypted") == QStringLiteral("1")) {
             _encryptionStatusCurrent = SyncFileItem::EncryptionStatus::EncryptedMigratedV2_0;
             Q_ASSERT(!_fileId.isEmpty());

src/libsync/discoveryphase.h

@@ -6,7 +6,7 @@
 
 #pragma once
 
 #include <QObject>
 #include <QElapsedTimer>
 #include <QStringList>
 #include <csync.h>
@@ -186,6 +186,7 @@
     void etag(const QByteArray &, const QDateTime &time);
     void finished(const OCC::HttpResult<QVector<OCC::RemoteInfo>> &result);
     void setfolderQuota(const FolderQuota &folderQuota);
+    void rootFileIdReceived(const qint64 fileId);
 
 private slots:
     void directoryListingIteratedSlot(const QString &, const QMap<QString, QString> &);

src/libsync/pushnotifications.cpp

@@ -1,3 +1,3 @@
 /*
  * SPDX-FileCopyrightText: 2021 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: GPL-2.0-or-later
@@ -7,9 +7,13 @@
 #include "creds/abstractcredentials.h"
 #include "account.h"
 
+#include <QJsonArray>
+
 namespace {
 static constexpr int MAX_ALLOWED_FAILED_AUTHENTICATION_ATTEMPTS = 3;
 static constexpr int PING_INTERVAL = 30 * 1000;
+
+static constexpr QLatin1String NOTIFY_FILE_ID_PREFIX = QLatin1String{"notify_file_id "};
 }
 
 namespace OCC {
@@ -102,7 +106,9 @@
 {
     qCInfo(lcPushNotifications) << "Received push notification:" << message;
 
-    if (message == "notify_file") {
+    if (message.startsWith(NOTIFY_FILE_ID_PREFIX)) {
+        handleNotifyFileId(message);
+    } else if (message == "notify_file") {
         handleNotifyFile();
     } else if (message == "notify_activity") {
         handleNotifyActivity();
@@ -124,7 +130,7 @@
         return;
     }
 
-    qCWarning(lcPushNotifications) << "Websocket error on with account" << _account->url() << error;
+    qCWarning(lcPushNotifications) << "Websocket error on with account" << _account->displayName() << _account->url() << error;
     closeWebSocket();
     emit connectionLost();
 }
@@ -153,7 +159,7 @@
 
 void PushNotifications::onWebSocketSslErrors(const QList<QSslError> &errors)
 {
-    qCWarning(lcPushNotifications) << "Websocket ssl errors on with account" << _account->url() << errors;
+    qCWarning(lcPushNotifications) << "Websocket ssl errors on with account" << _account->displayName() << _account->url() << errors;
     closeWebSocket();
     emit authenticationFailed();
 }
@@ -164,7 +170,7 @@
     const auto capabilities = _account->capabilities();
     const auto webSocketUrl = capabilities.pushNotificationsWebSocketUrl();
 
-    qCInfo(lcPushNotifications) << "Open connection to websocket on" << webSocketUrl << "for account" << _account->url();
+    qCInfo(lcPushNotifications) << "Open connection to websocket on" << webSocketUrl << "for account" << _account->displayName() << _account->url();
     connect(_webSocket, QOverload<QAbstractSocket::SocketError>::of(&QWebSocket::errorOccurred), this, &PushNotifications::onWebSocketError);
     connect(_webSocket, &QWebSocket::sslErrors, this, &PushNotifications::onWebSocketSslErrors);
     _webSocket->open(webSocketUrl);
@@ -184,6 +190,8 @@
 {
     qCInfo(lcPushNotifications) << "Authenticated successful on websocket";
     _failedAuthenticationAttemptsCount = 0;
+    qCDebug(lcPushNotifications) << "Requesting opt-in to 'notify_file_id' notifications";
+    _webSocket->sendTextMessage("listen notify_file_id");
     _isReady = true;
     startPingTimer();
     emit ready();
@@ -202,6 +210,35 @@
     emitFilesChanged();
 }
 
+void PushNotifications::handleNotifyFileId(const QString &message)
+{
+    qCDebug(lcPushNotifications) << "File-ID push notification arrived";
+
+    QList<qint64> fileIds{};
+    QJsonParseError parseError;
+    const auto fileIdsJson = message.mid(NOTIFY_FILE_ID_PREFIX.length());
+    const auto jsonDoc = QJsonDocument::fromJson(fileIdsJson.toUtf8(), &parseError);
+
+    if (parseError.error != QJsonParseError::NoError) {
+        qCWarning(lcPushNotifications).nospace() << "could not parse received list of file IDs error=" << parseError.error << " errorString=" << parseError.errorString() << " fileIdsJson=" << fileIdsJson;
+        return;
+    }
+
+    if (const auto jsonArray = jsonDoc.array(); jsonDoc.isArray()) {
+        for (const auto& fileid : jsonArray) {
+            if (const auto fid = fileid.toInteger(); fileid.isDouble()) {
+                fileIds.push_back(fid);
+            }
+        }
+    }
+
+    if (fileIds.empty()) {
+        return;
+    }
+
+    emit fileIdsChanged(_account, fileIds);
+}
+
 void PushNotifications::handleInvalidCredentials()
 {
     qCInfo(lcPushNotifications) << "Invalid credentials submitted to websocket";

src/libsync/pushnotifications.h

@@ -5,7 +5,7 @@
 
 #pragma once
 
 #include <QWebSocket>
 #include <QTimer>
 
 #include "capabilities.h"
@@ -65,6 +65,11 @@
      */
     void filesChanged(OCC::Account *account);
 
+    /**
+     * Will be emitted if specific files on the server changed
+     */
+    void fileIdsChanged(OCC::Account *account, const QList<qint64> &fileIds);
+
     /**
      * Will be emitted if activities have been changed on the server
      */
@@ -111,6 +116,7 @@
 
     void handleAuthenticated();
     void handleNotifyFile();
+    void handleNotifyFileId(const QString &message);
     void handleInvalidCredentials();
     void handleNotifyNotification();
     void handleNotifyActivity();

src/libsync/syncengine.cpp

@@ -761,6 +761,7 @@ void SyncEngine::startSync()
     _discoveryPhase->startJob(discoveryJob);
     connect(discoveryJob, &ProcessDirectoryJob::etag, this, &SyncEngine::slotRootEtagReceived);
     connect(discoveryJob, &ProcessDirectoryJob::updatedRootFolderQuota, account().data(), &Account::rootFolderQuotaChanged);
+    connect(discoveryJob, &ProcessDirectoryJob::rootFileIdReceived, this, &SyncEngine::slotRootFileIdReceived);
     connect(_discoveryPhase.get(), &DiscoveryPhase::addErrorToGui, this, &SyncEngine::addErrorToGui);
 }
 
@@ -792,6 +793,16 @@ void SyncEngine::slotRootEtagReceived(const QByteArray &e, const QDateTime &time
     }
 }
 
+void SyncEngine::slotRootFileIdReceived(const qint64 fileId)
+{
+    if (_rootFileIdReceived) {
+        return;
+    }
+    _rootFileId = fileId;
+    _rootFileIdReceived = true;
+    emit rootFileIdReceived(fileId);
+}
+
 void SyncEngine::slotNewItem(const SyncFileItemPtr &item)
 {
     _progressInfo->adjustTotalsForFile(*item);

src/libsync/syncengine.h

@@ -6,7 +6,7 @@
 
 #pragma once
 
 #include <cstdint>
 
 #include <QMutex>
 #include <QThread>
@@ -155,6 +155,7 @@
 signals:
     // During update, before reconcile
     void rootEtag(const QByteArray &, const QDateTime &);
+    void rootFileIdReceived(const qint64 fileId);
 
     // after the above signals. with the items that actually need propagating
     void aboutToPropagate(OCC::SyncFileItemVector &);
@@ -197,6 +198,7 @@
 private slots:
     void slotFolderDiscovered(bool local, const QString &folder);
     void slotRootEtagReceived(const QByteArray &, const QDateTime &time);
+    void slotRootFileIdReceived(const qint64 fileId);
 
     /** When the discovery phase discovers an item */
     void slotItemDiscovered(const OCC::SyncFileItemPtr &item);
@@ -326,6 +328,8 @@
     QString _localPath;
     QString _remotePath;
     QByteArray _remoteRootEtag;
+    bool _rootFileIdReceived = false;
+    qint64 _rootFileId = 0;
     SyncJournalDb *_journal;
     std::unique_ptr<DiscoveryPhase> _discoveryPhase;
     QSharedPointer<OwncloudPropagator> _propagator;

test/pushnotificationstestutils.cpp

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: GPL-2.0-or-later
  */
 
 #include <QLoggingCategory>
 #include <QSignalSpy>
 #include <QTest>
 #include <cstdint>
@@ -67,6 +67,11 @@
         return nullptr;
     }
 
+    // Wait for notify_file_id opt-in
+    if (!waitForTextMessages()) {
+        return nullptr;
+    }
+
     afterAuthentication();
 
     return socket;