mirrors/nextcloud-desktop
3
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2025-10-26 11:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

New Discovery Algo: Support the DatabaseAndFilesystem mode for local discovery

Browse Source
This commit is contained in:
Olivier Goffart 2018-07-18 13:44:41 +02:00
parent 10b8ca640a
commit 0905892835
7 changed files with 48 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/gui/issueswidget.cpp
View File

@ -261,7 +261,7 @@ void IssuesWidget::slotProgressInfo(const QString &folder, const ProgressInfo &p
if (!QFileInfo(f->path() + ProtocolItem::extraData(item).path).exists())
return true;
auto path = QFileInfo(ProtocolItem::extraData(item).path).dir().path().toUtf8();
auto path = QFileInfo(ProtocolItem::extraData(item).path).dir().path();
if (path == ".")
path.clear();

44
src/libsync/discovery.cpp
View File

@ -83,6 +83,13 @@ void ProcessDirectoryJob::start()
_hasServerEntries = true;
}
if (_queryLocal == NormalQuery) {
if (!_discoveryData->_shouldDiscoverLocaly(_currentFolder._local)
&& (_currentFolder._local == _currentFolder._original || !_discoveryData->_shouldDiscoverLocaly(_currentFolder._original))) {
_queryLocal = ParentNotChanged;
}
}
if (_queryLocal == NormalQuery) {
/*QDirIterator dirIt(_propagator->_localDir + _currentFolder);
while (dirIt.hasNext()) {
@ -187,14 +194,14 @@ void ProcessDirectoryJob::process()
}
_localEntries.clear();
if (_queryServer == ParentNotChanged) {
if (_queryServer == ParentNotChanged || _queryLocal == ParentNotChanged) {
// fetch all the name from the DB
auto pathU8 = _currentFolder._original.toUtf8();
// FIXME cache, and do that better (a query that do not get stuff recursively)
if (!_discoveryData->_statedb->getFilesBelowPath(pathU8, [&](const SyncJournalFileRecord &rec) {
if (rec._path.indexOf("/", pathU8.size() + 1) > 0)
return;
auto name = QString::fromUtf8(rec._path.mid(pathU8.size() + 1));
auto name = pathU8.isEmpty() ? rec._path : QString::fromUtf8(rec._path.mid(pathU8.size() + 1));
if (rec._type == ItemTypeVirtualFile || rec._type == ItemTypeVirtualFileDownload) {
name.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
}
@ -205,15 +212,21 @@ void ProcessDirectoryJob::process()
}
}
for (const auto &f : entriesNames) {
auto localEntry = localEntriesHash.value(f);
auto serverEntry = serverEntriesHash.value(f);
SyncJournalFileRecord record = dbEntriesHash.value(f);
PathTuple path;
if ((localEntry.isValid() && localEntry.isVirtualFile)) {
Q_ASSERT(localEntry.name.endsWith(_discoveryData->_syncOptions._virtualFileSuffix));
path = _currentFolder.addName(localEntry.name);
path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
} else if (_queryLocal == ParentNotChanged && record.isValid() && record._type == ItemTypeVirtualFile) {
QString name = f + _discoveryData->_syncOptions._virtualFileSuffix;
path = _currentFolder.addName(name);
path._server.chop(_discoveryData->_syncOptions._virtualFileSuffix.size());
} else {
path = _currentFolder.addName(f);
}
@ -226,8 +239,7 @@ void ProcessDirectoryJob::process()
if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden))
continue;
SyncJournalFileRecord record = dbEntriesHash[f];
if (_queryServer != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) {
if (_queryServer != ParentNotChanged && _queryLocal != ParentNotChanged && !_discoveryData->_statedb->getFileRecord(path._original, &record)) {
qFatal("TODO: DB ERROR HANDLING");
}
if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._original)) {
@ -358,8 +370,9 @@ void ProcessDirectoryJob::processFile(PathTuple path,
const SyncJournalFileRecord &dbEntry)
{
const char *hasServer = serverEntry.isValid() ? "true" : _queryServer == ParentNotChanged ? "db" : "false";
const char *hasLocal = localEntry.isValid() ? "true" : _queryLocal == ParentNotChanged ? "db" : "false";
qCInfo(lcDisco).nospace() << "Processing " << path._original
<< " | valid: " << dbEntry.isValid() << "/" << localEntry.isValid() << "/" << hasServer
<< " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer
<< " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime
<< " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size
<< " | etag: " << dbEntry._etag << "//" << serverEntry.etag
@ -588,7 +601,7 @@ void ProcessDirectoryJob::processFile(PathTuple path,
item->_size = serverEntry.size;
if (serverEntry.isDirectory && dbEntry._type == ItemTypeDirectory) {
item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
} else if (!localEntry.isValid()) {
} else if (!localEntry.isValid() && _queryLocal != ParentNotChanged) {
// Deleted locally, changed on server
item->_instruction = CSYNC_INSTRUCTION_NEW;
} else {
@ -850,6 +863,12 @@ void ProcessDirectoryJob::processFile(PathTuple path,
}
}
}
} else if (_queryLocal == ParentNotChanged && dbEntry.isValid()) {
if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
// Not modified locally (ParentNotChanged), bit not on the server: Removed on the server.
item->_instruction = CSYNC_INSTRUCTION_REMOVE;
item->_direction = SyncFileItem::Down;
}
} else if (_queryServer != ParentNotChanged && !serverEntry.isValid()) {
// Not locally, not on the server. The entry is stale!
qCInfo(lcDisco) << "Stale DB entry";
@ -878,12 +897,15 @@ void ProcessDirectoryJob::processFile(PathTuple path,
qCInfo(lcDisco) << "Discovered" << item->_file << item->_instruction << item->_direction << item->_type;
if (item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory) {
if (item->_instruction == CSYNC_INSTRUCTION_SYNC) {
item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
}
if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_SYNC)
item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
bool recurse = item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory;
if (_queryLocal != NormalQuery && _queryServer != NormalQuery)
recurse = false;
if (recurse) {
auto job = new ProcessDirectoryJob(item, recurseQueryServer,
localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
_queryLocal == ParentNotChanged ? ParentNotChanged : localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist,
_discoveryData, this);
job->_currentFolder = path;
if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) {

1
src/libsync/discoveryphase.h
View File

@ -118,6 +118,7 @@ public:
ExcludedFiles *_excludes;
QString _invalidFilenamePattern; // FIXME: maybe move in ExcludedFiles
bool _ignoreHiddenFiles = false;
std::function<bool(const QString &)> _shouldDiscoverLocaly;
bool isInSelectiveSyncBlackList(const QString &path) const;
bool checkSelectiveSyncNewFolder(const QString &path, RemotePermissions rp);

6
src/libsync/localdiscoverytracker.cpp
View File

@ -26,7 +26,7 @@ LocalDiscoveryTracker::LocalDiscoveryTracker()
{
}
void LocalDiscoveryTracker::addTouchedPath(const QByteArray &relativePath)
void LocalDiscoveryTracker::addTouchedPath(const QString &relativePath)
{
qCDebug(lcLocalDiscoveryTracker) << "inserted touched" << relativePath;
_localDiscoveryPaths.insert(relativePath);
@ -42,7 +42,7 @@ void LocalDiscoveryTracker::startSyncFullDiscovery()
void LocalDiscoveryTracker::startSyncPartialDiscovery()
{
if (lcLocalDiscoveryTracker().isDebugEnabled()) {
QByteArrayList paths;
QStringList paths;
for (auto &path : _localDiscoveryPaths)
paths.append(path);
qCDebug(lcLocalDiscoveryTracker) << "partial discovery with paths: " << paths;
@ -52,7 +52,7 @@ void LocalDiscoveryTracker::startSyncPartialDiscovery()
_localDiscoveryPaths.clear();
}
const std::set<QByteArray> &LocalDiscoveryTracker::localDiscoveryPaths() const
const std::set<QString> &LocalDiscoveryTracker::localDiscoveryPaths() const
{
return _localDiscoveryPaths;
}

8
src/libsync/localdiscoverytracker.h
View File

@ -57,7 +57,7 @@ public:
* This should be a full relative file path, example:
* foo/bar/file.txt
*/
void addTouchedPath(const QByteArray &relativePath);
void addTouchedPath(const QString &relativePath);
/** Call when a sync run starts that rediscovers all local files */
void startSyncFullDiscovery();
@ -66,7 +66,7 @@ public:
void startSyncPartialDiscovery();
/** Access list of files that shall be locally rediscovered. */
const std::set<QByteArray> &localDiscoveryPaths() const;
const std::set<QString> &localDiscoveryPaths() const;
public slots:
/**
@ -87,7 +87,7 @@ private:
* Mostly a collection of files the filewatchers have reported as touched.
* Also includes files that have had errors in the last sync run.
*/
std::set<QByteArray> _localDiscoveryPaths;
std::set<QString> _localDiscoveryPaths;
/**
* The paths that the current sync run used for local discovery.
@ -95,7 +95,7 @@ private:
* For failing syncs, this list will be merged into _localDiscoveryPaths
* again when the sync is done to make sure everything is retried.
*/
std::set<QByteArray> _previousLocalDiscoveryPaths;
std::set<QString> _previousLocalDiscoveryPaths;
};
} // namespace OCC

8
src/libsync/syncengine.cpp
View File

@ -756,9 +756,6 @@ void SyncEngine::startSync()
_csync_ctx->read_remote_from_db = true;
_lastLocalDiscoveryStyle = _localDiscoveryStyle;
_csync_ctx->should_discover_locally_fn = [this](const QByteArray &path) {
return shouldDiscoverLocally(path);
};
_csync_ctx->new_files_are_virtual = _syncOptions._newFilesAreVirtual;
_csync_ctx->virtual_file_suffix = _syncOptions._virtualFileSuffix.toUtf8();
@ -804,6 +801,7 @@ void SyncEngine::startSync()
_discoveryPhase->_syncOptions = _syncOptions;
_discoveryPhase->_selectiveSyncBlackList = selectiveSyncBlackList;
_discoveryPhase->_selectiveSyncWhiteList = _journal->getSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, &ok);
_discoveryPhase->_shouldDiscoverLocaly = [this](const QString &s) { return shouldDiscoverLocally(s); };
if (!ok) {
qCWarning(lcEngine) << "Unable to read selective sync list, aborting.";
csyncError(tr("Unable to read from the sync journal."));
@ -1442,13 +1440,13 @@ AccountPtr SyncEngine::account() const
return _account;
}
void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QByteArray> paths)
void SyncEngine::setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QString> paths)
{
_localDiscoveryStyle = style;
_localDiscoveryPaths = std::move(paths);
}
bool SyncEngine::shouldDiscoverLocally(const QByteArray &path) const
bool SyncEngine::shouldDiscoverLocally(const QString &path) const
{
if (_localDiscoveryStyle == LocalDiscoveryStyle::FilesystemOnly)
return true;

6
src/libsync/syncengine.h
View File

@ -113,7 +113,7 @@ public:
* revert afterwards. Use _lastLocalDiscoveryStyle to discover the last
* sync's style.
*/
void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QByteArray> paths = {});
void setLocalDiscoveryOptions(LocalDiscoveryStyle style, std::set<QString> paths = {});
/**
* Returns whether the given folder-relative path should be locally discovered
@ -122,7 +122,7 @@ public:
* Example: If path is 'foo/bar' and style is DatabaseAndFilesystem and dirs contains
* 'foo/bar/touched_file', then the result will be true.
*/
bool shouldDiscoverLocally(const QByteArray &path) const;
bool shouldDiscoverLocally(const QString &path) const;
/** Access the last sync run's local discovery style */
LocalDiscoveryStyle lastLocalDiscoveryStyle() const { return _lastLocalDiscoveryStyle; }
@ -309,7 +309,7 @@ private:
/** The kind of local discovery the last sync run used */
LocalDiscoveryStyle _lastLocalDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly;
LocalDiscoveryStyle _localDiscoveryStyle = LocalDiscoveryStyle::FilesystemOnly;
std::set<QByteArray> _localDiscoveryPaths;
std::set<QString> _localDiscoveryPaths;
};
}