From 40ecd90abf69d90ca0edc1de93bc0173edad632b Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Mon, 11 Aug 2025 16:51:03 +0200 Subject: [PATCH 1/9] fix(quota): return unlimited free space for type changes. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 919d176892..118e31b346 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1085,7 +1085,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it int64_t ProcessDirectoryJob::folderBytesAvailable(const SyncFileItemPtr &item, const FolderQuota::ServerEntry serverEntry) const { const auto unlimitedFreeSpace = -3; - if (item->_size == 0 || item->_direction != SyncFileItem::Up || item->isDirectory()) { + if (item->_size == 0 || item->_direction != SyncFileItem::Up || item->isDirectory() || item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE) { return unlimitedFreeSpace; } @@ -1093,8 +1093,7 @@ int64_t ProcessDirectoryJob::folderBytesAvailable(const SyncFileItemPtr &item, c return unlimitedFreeSpace; } - if (item->_instruction != CSYNC_INSTRUCTION_TYPE_CHANGE - && (serverEntry == FolderQuota::ServerEntry::Valid || !_dirItem)) { + if (serverEntry == FolderQuota::ServerEntry::Valid || !_dirItem) { return _folderQuota.bytesAvailable; } From 1fc1d7be4f50d9066294f239ac651ccc6b9257fa Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Mon, 11 Aug 2025 18:58:41 +0200 Subject: [PATCH 2/9] feat(syncjournaldb): add default value to quotaBytesAvailable and quotaBytesUsed columns. Fix for #8555. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index a94d7e017e..eb32073e49 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -723,11 +723,15 @@ bool SyncJournalDb::updateMetadataTableStructure() return false; } - const auto addColumn = [this, &columns, &re] (const QString &columnName, const QString &dataType, const bool withIndex = false) { + const auto addColumn = [this, &columns, &re] (const QString &columnName, const QString &dataType, const bool withIndex = false, const QString defaultCommand = {}) { const auto latin1ColumnName = columnName.toLatin1(); if (columns.indexOf(latin1ColumnName) == -1) { SqlQuery query(_db); - const auto request = QStringLiteral("ALTER TABLE metadata ADD COLUMN %1 %2;").arg(columnName).arg(dataType); + auto request = QStringLiteral("ALTER TABLE metadata ADD COLUMN %1 %2").arg(columnName).arg(dataType); + if (!defaultCommand.isEmpty()) { + request.append(QStringLiteral(" ") + defaultCommand); + } + request.append(QStringLiteral(";")); query.prepare(request.toLatin1()); if (!query.exec()) { sqlFail(QStringLiteral("updateMetadataTableStructure: add %1 column").arg(columnName), query); @@ -851,8 +855,8 @@ bool SyncJournalDb::updateMetadataTableStructure() addColumn(QStringLiteral("isLivePhoto"), QStringLiteral("INTEGER")); addColumn(QStringLiteral("livePhotoFile"), QStringLiteral("TEXT")); - addColumn(QStringLiteral("quotaBytesUsed"), QStringLiteral("BIGINT")); - addColumn(QStringLiteral("quotaBytesAvailable"), QStringLiteral("BIGINT")); + addColumn(QStringLiteral("quotaBytesUsed"), QStringLiteral("BIGINT"), false, QStringLiteral("DEFAULT -1 NOT NULL")); + addColumn(QStringLiteral("quotaBytesAvailable"), QStringLiteral("BIGINT"), false, QStringLiteral("DEFAULT -1 NOT NULL")); return re; } From 95d7e54f7b2ae5e62345d33ccef5d18523c8a77c Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Mon, 11 Aug 2025 20:45:02 +0200 Subject: [PATCH 3/9] test(syncjournaldb): folder quota default values. Signed-off-by: Camila Ayres --- test/testsyncjournaldb.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/testsyncjournaldb.cpp b/test/testsyncjournaldb.cpp index 7c5febf12b..d62ae248bd 100644 --- a/test/testsyncjournaldb.cpp +++ b/test/testsyncjournaldb.cpp @@ -98,6 +98,35 @@ private slots: QVERIFY(!record.isValid()); } + void testFolderQuota() + { + const auto bigFolderRecord = QByteArray("bigfolder"); + SyncJournalFileRecord record; + record._path = bigFolderRecord; + record._inode = std::numeric_limits::max() + 12ull; + record._modtime = dropMsecs(QDateTime::currentDateTime()); + record._type = ItemTypeDirectory; + record._etag = "123123"; + record._fileId = "abcd"; + record._fileSize = 213089999; + QVERIFY(_db.setFileRecord(record)); + + SyncJournalFileRecord storedRecord; + QVERIFY(_db.getFileRecord(bigFolderRecord, &storedRecord)); + QVERIFY(storedRecord == record); + // default values + QCOMPARE(storedRecord._folderQuota.bytesAvailable, -1); + QCOMPARE(storedRecord._folderQuota.bytesUsed, -1); + + record._folderQuota.bytesUsed = 100; + record._folderQuota.bytesAvailable = 5000; + QVERIFY(_db.setFileRecord(record)); + QVERIFY(_db.getFileRecord(bigFolderRecord, &storedRecord)); + QVERIFY(storedRecord == record); + QCOMPARE(storedRecord._folderQuota.bytesAvailable, 5000); + QCOMPARE(storedRecord._folderQuota.bytesUsed, 100); + } + void testFileRecordChecksum() { // Try with and without a checksum From af3ee7c2982b77e1936d5203252b3b5af0a3c94b Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 12 Aug 2025 13:11:01 +0200 Subject: [PATCH 4/9] feat(quota): add default value to metadata column if it doesn't exist. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.cpp | 61 ++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index eb32073e49..d5b84c1259 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -714,7 +714,6 @@ bool SyncJournalDb::updateDatabaseStructure() bool SyncJournalDb::updateMetadataTableStructure() { - auto columns = tableColumns("metadata"); bool re = true; @@ -723,9 +722,44 @@ bool SyncJournalDb::updateMetadataTableStructure() return false; } - const auto addColumn = [this, &columns, &re] (const QString &columnName, const QString &dataType, const bool withIndex = false, const QString defaultCommand = {}) { - const auto latin1ColumnName = columnName.toLatin1(); - if (columns.indexOf(latin1ColumnName) == -1) { + const auto columnExists = [&columns] (const QString &columnName) -> bool { + return columns.indexOf(columnName.toLatin1()) > -1; + }; + + const auto hasDefaultSet = [this] (const QString &columnName) -> bool { + SqlQuery query(_db); + const auto selectDefault = QStringLiteral("SELECT dflt_value FROM pragma_table_info('metadata') WHERE name = '%1';").arg(columnName); + query.prepare(selectDefault.toLatin1()); + if (!query.exec()) { + sqlFail(QStringLiteral("check default value for: %1").arg(columnName), query); + return false; + } + + if (const auto result = query.next();!result.ok || !result.hasData) { + qCWarning(lcDb) << "database error:" << query.error(); + return false; + } + + return !query.nullValue(0); + }; + + const auto removeColumn = [this, &re, &columns] (const QString &columnName) -> bool { + SqlQuery query(_db); + const auto request = QStringLiteral("ALTER TABLE metadata DROP COLUMN %1;").arg(columnName); + query.prepare(request.toLatin1()); + if (!query.exec()) { + sqlFail(QStringLiteral("update metadata structure: drop %1 column").arg(columnName), query); + re = false; + return false; + } + //update list + columns = tableColumns("metadata"); + commitInternal(QStringLiteral("update database structure: drop %1 column").arg(columnName)); + return true; + }; + + const auto addColumn = [this, &re, &columnExists] (const QString &columnName, const QString &dataType, const bool withIndex = false, const QString defaultCommand = {}) { + if (!columnExists(columnName)) { SqlQuery query(_db); auto request = QStringLiteral("ALTER TABLE metadata ADD COLUMN %1 %2").arg(columnName).arg(dataType); if (!defaultCommand.isEmpty()) { @@ -855,8 +889,23 @@ bool SyncJournalDb::updateMetadataTableStructure() addColumn(QStringLiteral("isLivePhoto"), QStringLiteral("INTEGER")); addColumn(QStringLiteral("livePhotoFile"), QStringLiteral("TEXT")); - addColumn(QStringLiteral("quotaBytesUsed"), QStringLiteral("BIGINT"), false, QStringLiteral("DEFAULT -1 NOT NULL")); - addColumn(QStringLiteral("quotaBytesAvailable"), QStringLiteral("BIGINT"), false, QStringLiteral("DEFAULT -1 NOT NULL")); + + { + const auto quotaBytesUsed = QStringLiteral("quotaBytesUsed"); + const auto quotaBytesAvailable = QStringLiteral("quotaBytesAvailable"); + const auto defaultCommand = QStringLiteral("DEFAULT -1 NOT NULL"); + const auto bigInt = QStringLiteral("BIGINT"); + + if (columnExists(quotaBytesUsed) && !hasDefaultSet(quotaBytesUsed)) { + removeColumn(quotaBytesUsed); + } + addColumn(quotaBytesUsed, bigInt, false, defaultCommand); + + if (columnExists(quotaBytesAvailable) && !hasDefaultSet(quotaBytesAvailable)) { + removeColumn(quotaBytesAvailable); + } + addColumn(quotaBytesAvailable, bigInt, false, defaultCommand); + } return re; } From de4f607837ca61648f44251db70f20cfe888bf1d Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 12 Aug 2025 22:31:02 +0200 Subject: [PATCH 5/9] refactor(syncjournaldb): move removeColumn lambda to a function. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.cpp | 37 ++++++++++++++++++------------------ src/common/syncjournaldb.h | 1 + 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index d5b84c1259..4f4d104d5b 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -712,6 +712,20 @@ bool SyncJournalDb::updateDatabaseStructure() return true; } +bool SyncJournalDb::removeColumn(const QString &columnName) +{ + SqlQuery query(_db); + const auto request = QStringLiteral("ALTER TABLE metadata DROP COLUMN %1;").arg(columnName); + query.prepare(request.toLatin1()); + if (!query.exec()) { + sqlFail(QStringLiteral("update metadata structure: drop %1 column").arg(columnName), query); + return false; + } + + commitInternal(QStringLiteral("update database structure: drop %1 column").arg(columnName)); + return true; +} + bool SyncJournalDb::updateMetadataTableStructure() { auto columns = tableColumns("metadata"); @@ -743,21 +757,6 @@ bool SyncJournalDb::updateMetadataTableStructure() return !query.nullValue(0); }; - const auto removeColumn = [this, &re, &columns] (const QString &columnName) -> bool { - SqlQuery query(_db); - const auto request = QStringLiteral("ALTER TABLE metadata DROP COLUMN %1;").arg(columnName); - query.prepare(request.toLatin1()); - if (!query.exec()) { - sqlFail(QStringLiteral("update metadata structure: drop %1 column").arg(columnName), query); - re = false; - return false; - } - //update list - columns = tableColumns("metadata"); - commitInternal(QStringLiteral("update database structure: drop %1 column").arg(columnName)); - return true; - }; - const auto addColumn = [this, &re, &columnExists] (const QString &columnName, const QString &dataType, const bool withIndex = false, const QString defaultCommand = {}) { if (!columnExists(columnName)) { SqlQuery query(_db); @@ -897,13 +896,15 @@ bool SyncJournalDb::updateMetadataTableStructure() const auto bigInt = QStringLiteral("BIGINT"); if (columnExists(quotaBytesUsed) && !hasDefaultSet(quotaBytesUsed)) { - removeColumn(quotaBytesUsed); + re = removeColumn(quotaBytesUsed); } - addColumn(quotaBytesUsed, bigInt, false, defaultCommand); if (columnExists(quotaBytesAvailable) && !hasDefaultSet(quotaBytesAvailable)) { - removeColumn(quotaBytesAvailable); + re = removeColumn(quotaBytesAvailable); } + + columns = tableColumns("metadata"); + addColumn(quotaBytesUsed, bigInt, false, defaultCommand); addColumn(quotaBytesAvailable, bigInt, false, defaultCommand); } diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 71ceb470c5..e15201f09a 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -399,6 +399,7 @@ private: [[nodiscard]] bool updateDatabaseStructure(); [[nodiscard]] bool updateMetadataTableStructure(); [[nodiscard]] bool updateErrorBlacklistTableStructure(); + [[nodiscard]] bool removeColumn(const QString &columnName); bool sqlFail(const QString &log, const SqlQuery &query); void commitInternal(const QString &context, bool startTrans = true); void startTransaction(); From 0a320a5960745031fda612a29f916bd0304515b8 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 12 Aug 2025 22:43:59 +0200 Subject: [PATCH 6/9] refactor(syncjournaldb): move hasDefaultValue lambda to a function. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.cpp | 39 ++++++++++++++++++------------------ src/common/syncjournaldb.h | 1 + 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 4f4d104d5b..dc9b79de66 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -712,6 +712,24 @@ bool SyncJournalDb::updateDatabaseStructure() return true; } +bool SyncJournalDb::hasDefaultValue(const QString &columnName) +{ + SqlQuery query(_db); + const auto selectDefault = QStringLiteral("SELECT dflt_value FROM pragma_table_info('metadata') WHERE name = '%1';").arg(columnName); + query.prepare(selectDefault.toLatin1()); + if (!query.exec()) { + sqlFail(QStringLiteral("check default value for: %1").arg(columnName), query); + return false; + } + + if (const auto result = query.next();!result.ok || !result.hasData) { + qCWarning(lcDb) << "database error:" << query.error(); + return false; + } + + return !query.nullValue(0); +} + bool SyncJournalDb::removeColumn(const QString &columnName) { SqlQuery query(_db); @@ -740,23 +758,6 @@ bool SyncJournalDb::updateMetadataTableStructure() return columns.indexOf(columnName.toLatin1()) > -1; }; - const auto hasDefaultSet = [this] (const QString &columnName) -> bool { - SqlQuery query(_db); - const auto selectDefault = QStringLiteral("SELECT dflt_value FROM pragma_table_info('metadata') WHERE name = '%1';").arg(columnName); - query.prepare(selectDefault.toLatin1()); - if (!query.exec()) { - sqlFail(QStringLiteral("check default value for: %1").arg(columnName), query); - return false; - } - - if (const auto result = query.next();!result.ok || !result.hasData) { - qCWarning(lcDb) << "database error:" << query.error(); - return false; - } - - return !query.nullValue(0); - }; - const auto addColumn = [this, &re, &columnExists] (const QString &columnName, const QString &dataType, const bool withIndex = false, const QString defaultCommand = {}) { if (!columnExists(columnName)) { SqlQuery query(_db); @@ -895,11 +896,11 @@ bool SyncJournalDb::updateMetadataTableStructure() const auto defaultCommand = QStringLiteral("DEFAULT -1 NOT NULL"); const auto bigInt = QStringLiteral("BIGINT"); - if (columnExists(quotaBytesUsed) && !hasDefaultSet(quotaBytesUsed)) { + if (columnExists(quotaBytesUsed) && !hasDefaultValue(quotaBytesUsed)) { re = removeColumn(quotaBytesUsed); } - if (columnExists(quotaBytesAvailable) && !hasDefaultSet(quotaBytesAvailable)) { + if (columnExists(quotaBytesAvailable) && !hasDefaultValue(quotaBytesAvailable)) { re = removeColumn(quotaBytesAvailable); } diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index e15201f09a..5c270b8061 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -400,6 +400,7 @@ private: [[nodiscard]] bool updateMetadataTableStructure(); [[nodiscard]] bool updateErrorBlacklistTableStructure(); [[nodiscard]] bool removeColumn(const QString &columnName); + [[nodiscard]] bool hasDefaultValue(const QString &columnName); bool sqlFail(const QString &log, const SqlQuery &query); void commitInternal(const QString &context, bool startTrans = true); void startTransaction(); From cd2fc994978f7af554b2499674b767c76dcc2d76 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 12 Aug 2025 22:50:59 +0200 Subject: [PATCH 7/9] refactor(syncjournaldb): refactor logic before updating the list of columns when removing one. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index dc9b79de66..dc12518479 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -895,16 +895,20 @@ bool SyncJournalDb::updateMetadataTableStructure() const auto quotaBytesAvailable = QStringLiteral("quotaBytesAvailable"); const auto defaultCommand = QStringLiteral("DEFAULT -1 NOT NULL"); const auto bigInt = QStringLiteral("BIGINT"); + auto result = false; if (columnExists(quotaBytesUsed) && !hasDefaultValue(quotaBytesUsed)) { - re = removeColumn(quotaBytesUsed); + result = removeColumn(quotaBytesUsed); } if (columnExists(quotaBytesAvailable) && !hasDefaultValue(quotaBytesAvailable)) { - re = removeColumn(quotaBytesAvailable); + result = removeColumn(quotaBytesAvailable); + } + + if (result) { + columns = tableColumns("metadata"); } - columns = tableColumns("metadata"); addColumn(quotaBytesUsed, bigInt, false, defaultCommand); addColumn(quotaBytesAvailable, bigInt, false, defaultCommand); } From 17001c60f9799fe8b2ebdd929a5b7b682d18cdf9 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Wed, 13 Aug 2025 10:28:27 +0200 Subject: [PATCH 8/9] test(syncjournaldb): add test for migrating folder quota columns. For upgrading from 3.17 release candidate to final. Signed-off-by: Camila Ayres --- src/common/syncjournaldb.h | 4 ++++ test/testsyncjournaldb.cpp | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 5c270b8061..14ba342227 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -21,6 +21,8 @@ #include "common/result.h" #include "common/pinstate.h" +class TestSyncJournalDB; + namespace OCC { class SyncJournalFileRecord; @@ -446,6 +448,8 @@ private: QByteArray _journalMode; PreparedSqlQueryManager _queryManager; + + friend class ::TestSyncJournalDB; }; bool OCSYNC_EXPORT diff --git a/test/testsyncjournaldb.cpp b/test/testsyncjournaldb.cpp index d62ae248bd..bbc62e1bfb 100644 --- a/test/testsyncjournaldb.cpp +++ b/test/testsyncjournaldb.cpp @@ -127,6 +127,23 @@ private slots: QCOMPARE(storedRecord._folderQuota.bytesUsed, 100); } + void testFolderMigration() { + const auto quotaBytesUsed = QStringLiteral("quotaBytesUsed"); + const auto quotaBytesAvailable = QStringLiteral("quotaBytesAvailable"); + const auto metadata = QStringLiteral("metadata"); + const auto columnsBeforeRemoval = _db.tableColumns(metadata.toLatin1()); + QVERIFY(columnsBeforeRemoval.indexOf(quotaBytesUsed.toLatin1()) > -1); + QVERIFY(columnsBeforeRemoval.indexOf(quotaBytesAvailable.toLatin1()) > -1); + QVERIFY(_db.removeColumn(quotaBytesAvailable)); + QVERIFY(_db.removeColumn(quotaBytesUsed)); + QVERIFY(_db.updateMetadataTableStructure()); + const auto columnsAfterConnect = _db.tableColumns(metadata.toLatin1()); + QVERIFY(columnsAfterConnect.indexOf(quotaBytesUsed.toLatin1()) > -1); + QVERIFY(columnsAfterConnect.indexOf(quotaBytesAvailable.toLatin1()) > -1); + QVERIFY(_db.hasDefaultValue(quotaBytesUsed)); + QVERIFY(_db.hasDefaultValue(quotaBytesAvailable)); + } + void testFileRecordChecksum() { // Try with and without a checksum From d28dbb7c2547334444306efca78841d6a72752c3 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Wed, 13 Aug 2025 10:53:20 +0200 Subject: [PATCH 9/9] test(syncjournaldb): use QCOMPARE_GT. Co-authored-by: Jyrki Gadinger Signed-off-by: Camila Ayres --- test/testsyncjournaldb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testsyncjournaldb.cpp b/test/testsyncjournaldb.cpp index bbc62e1bfb..424b439af3 100644 --- a/test/testsyncjournaldb.cpp +++ b/test/testsyncjournaldb.cpp @@ -132,8 +132,8 @@ private slots: const auto quotaBytesAvailable = QStringLiteral("quotaBytesAvailable"); const auto metadata = QStringLiteral("metadata"); const auto columnsBeforeRemoval = _db.tableColumns(metadata.toLatin1()); - QVERIFY(columnsBeforeRemoval.indexOf(quotaBytesUsed.toLatin1()) > -1); - QVERIFY(columnsBeforeRemoval.indexOf(quotaBytesAvailable.toLatin1()) > -1); + QCOMPARE_GT(columnsBeforeRemoval.indexOf(quotaBytesUsed.toLatin1()), -1); + QCOMPARE_GT(columnsBeforeRemoval.indexOf(quotaBytesAvailable.toLatin1()), -1); QVERIFY(_db.removeColumn(quotaBytesAvailable)); QVERIFY(_db.removeColumn(quotaBytesUsed)); QVERIFY(_db.updateMetadataTableStructure());