/* * Copyright (C) by Olivier Goffart * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. */ #include "discovery.h" #include "common/syncjournaldb.h" #include "syncfileitem.h" #include #include #include #include #include #include "vio/csync_vio_local.h" #include "common/checksums.h" #include "csync_exclude.h" #include "csync_util.h" namespace OCC { Q_LOGGING_CATEGORY(lcDisco, "sync.discovery", QtInfoMsg) void ProcessDirectoryJob::start() { qCInfo(lcDisco) << "STARTING" << _currentFolder._server << _queryServer << _currentFolder._local << _queryLocal; DiscoverySingleDirectoryJob *serverJob = nullptr; if (_queryServer == NormalQuery) { serverJob = startAsyncServerQuery(); } else { _serverQueryDone = true; } // Check whether a normal local query is even necessary if (_queryLocal == NormalQuery) { if (!_discoveryData->_shouldDiscoverLocaly(_currentFolder._local) && (_currentFolder._local == _currentFolder._original || !_discoveryData->_shouldDiscoverLocaly(_currentFolder._original))) { _queryLocal = ParentNotChanged; } } if (_queryLocal == NormalQuery) { if (!runLocalQuery() && serverJob) serverJob->abort(); } _localQueryDone = true; // Process is being called when both local and server entries are fetched. if (_serverQueryDone) process(); } void ProcessDirectoryJob::process() { ASSERT(_localQueryDone && _serverQueryDone); QString localDir; // // Build lookup tables for local, remote and db entries. // For suffix-virtual files, the key will always be the base file name // without the suffix. // std::set entriesNames; // sorted QHash serverEntriesHash; QHash localEntriesHash; QHash dbEntriesHash; for (auto &e : _serverNormalQueryEntries) { entriesNames.insert(e.name); serverEntriesHash[e.name] = std::move(e); } _serverNormalQueryEntries.clear(); for (auto &e : _localNormalQueryEntries) { // Remove the virtual file suffix auto name = e.name; if (hasVirtualFileSuffix(name)) { e.isVirtualFile = true; chopVirtualFileSuffix(name); if (localEntriesHash.contains(name)) continue; // If there is both a virtual file and a real file, we must keep the real file } entriesNames.insert(name); localEntriesHash[name] = std::move(e); } _localNormalQueryEntries.clear(); // fetch all the name from the DB auto pathU8 = _currentFolder._original.toUtf8(); if (!_discoveryData->_statedb->listFilesInPath(pathU8, [&](const SyncJournalFileRecord &rec) { auto name = pathU8.isEmpty() ? rec._path : QString::fromUtf8(rec._path.mid(pathU8.size() + 1)); if (rec.isVirtualFile()) chopVirtualFileSuffix(name); entriesNames.insert(name); dbEntriesHash[name] = rec; })) { dbError(); return; } // // Iterate over entries and process them // for (const auto &f : entriesNames) { auto localEntry = localEntriesHash.value(f); auto serverEntry = serverEntriesHash.value(f); SyncJournalFileRecord record = dbEntriesHash.value(f); PathTuple path; path = _currentFolder.addName(f); // If the file is virtual in the db, adjust path._original if (record.isVirtualFile()) { ASSERT(hasVirtualFileSuffix(record._path)); addVirtualFileSuffix(path._original); } else if (localEntry.isVirtualFile) { // We don't have a db entry - but it should be at this path addVirtualFileSuffix(path._original); } // If the file is virtual locally, adjust path._local if (localEntry.isVirtualFile) { ASSERT(hasVirtualFileSuffix(localEntry.name)); addVirtualFileSuffix(path._local); } else if (record.isVirtualFile() && _queryLocal == ParentNotChanged) { addVirtualFileSuffix(path._local); } // If the filename starts with a . we consider it a hidden file // For windows, the hidden state is also discovered within the vio // local stat function. // Recall file shall not be ignored (#4420) bool isHidden = localEntry.isHidden || (f[0] == '.' && f != QLatin1String(".sys.admin#recall#")); if (handleExcluded(path._target, localEntry.isDirectory || serverEntry.isDirectory, isHidden, localEntry.isSymLink)) continue; if (_queryServer == InBlackList || _discoveryData->isInSelectiveSyncBlackList(path._original)) { processBlacklisted(path, localEntry, record); continue; } processFile(std::move(path), localEntry, serverEntry, record); } QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); } bool ProcessDirectoryJob::handleExcluded(const QString &path, bool isDirectory, bool isHidden, bool isSymlink) { // FIXME! call directly, without char* conversion auto excluded = _discoveryData->_excludes->traversalPatternMatch(path.toUtf8(), isDirectory ? ItemTypeDirectory : ItemTypeFile); // FIXME: move to ExcludedFiles 's regexp ? bool isInvalidPattern = false; if (excluded == CSYNC_NOT_EXCLUDED && !_discoveryData->_invalidFilenameRx.isEmpty()) { if (path.contains(_discoveryData->_invalidFilenameRx)) { excluded = CSYNC_FILE_EXCLUDE_INVALID_CHAR; isInvalidPattern = true; } } if (excluded == CSYNC_NOT_EXCLUDED && _discoveryData->_ignoreHiddenFiles && isHidden) { excluded = CSYNC_FILE_EXCLUDE_HIDDEN; } auto localCodec = QTextCodec::codecForLocale(); if (localCodec->mibEnum() != 106) { // If the locale codec is not UTF-8, we must check that the filename from the server can // be encoded in the local file system. // // We cannot use QTextCodec::canEncode() since that can incorrectly return true, see // https://bugreports.qt.io/browse/QTBUG-6925. QTextEncoder encoder(localCodec, QTextCodec::ConvertInvalidToNull); if (encoder.fromUnicode(path).contains('\0')) { qCWarning(lcDisco) << "Cannot encode " << path << " to local encoding " << localCodec->name(); excluded = CSYNC_FILE_EXCLUDE_CANNOT_ENCODE; } } if (excluded == CSYNC_NOT_EXCLUDED && !isSymlink) { return false; } else if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED || excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) { return true; } auto item = SyncFileItemPtr::create(); item->_file = path; item->_originalFile = path; item->_instruction = CSYNC_INSTRUCTION_IGNORE; if (isSymlink) { /* Symbolic links are ignored. */ item->_errorString = tr("Symbolic links are not supported in syncing."); } else { switch (excluded) { case CSYNC_NOT_EXCLUDED: case CSYNC_FILE_SILENTLY_EXCLUDED: case CSYNC_FILE_EXCLUDE_AND_REMOVE: qFatal("These were handled earlier"); case CSYNC_FILE_EXCLUDE_LIST: item->_errorString = tr("File is listed on the ignore list."); break; case CSYNC_FILE_EXCLUDE_INVALID_CHAR: if (item->_file.endsWith('.')) { item->_errorString = tr("File names ending with a period are not supported on this file system."); } else { char invalid = '\0'; foreach (char x, QByteArray("\\:?*\"<>|")) { if (item->_file.contains(x)) { invalid = x; break; } } if (invalid) { item->_errorString = tr("File names containing the character '%1' are not supported on this file system.") .arg(QLatin1Char(invalid)); } if (isInvalidPattern) { item->_errorString = tr("File name contains at least one invalid character"); } else { item->_errorString = tr("The file name is a reserved name on this file system."); } } break; case CSYNC_FILE_EXCLUDE_TRAILING_SPACE: item->_errorString = tr("Filename contains trailing spaces."); break; case CSYNC_FILE_EXCLUDE_LONG_FILENAME: item->_errorString = tr("Filename is too long."); break; case CSYNC_FILE_EXCLUDE_HIDDEN: item->_errorString = tr("File/Folder is ignored because it's hidden."); break; case CSYNC_FILE_EXCLUDE_STAT_FAILED: item->_errorString = tr("Stat failed."); break; case CSYNC_FILE_EXCLUDE_CONFLICT: item->_errorString = tr("Conflict: Server version downloaded, local copy renamed and not uploaded."); item->_status = SyncFileItem::Conflict; break; case CSYNC_FILE_EXCLUDE_CANNOT_ENCODE: item->_errorString = tr("The filename cannot be encoded on your file system."); break; } } _childIgnored = true; emit _discoveryData->itemDiscovered(item); return true; } void ProcessDirectoryJob::processFile(PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry) { const char *hasServer = serverEntry.isValid() ? "true" : _queryServer == ParentNotChanged ? "db" : "false"; const char *hasLocal = localEntry.isValid() ? "true" : _queryLocal == ParentNotChanged ? "db" : "false"; qCInfo(lcDisco).nospace() << "Processing " << path._original << " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size << " | etag: " << dbEntry._etag << "//" << serverEntry.etag << " | checksum: " << dbEntry._checksumHeader << "//" << serverEntry.checksumHeader << " | perm: " << dbEntry._remotePerm << "//" << serverEntry.remotePerm << " | fileid: " << dbEntry._fileId << "//" << serverEntry.fileId << " | inode: " << dbEntry._inode << "/" << localEntry.inode << "/"; if (_discoveryData->_renamedItems.contains(path._original)) { qCDebug(lcDisco) << "Ignoring renamed"; return; // Ignore this. } auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); item->_file = path._target; item->_originalFile = path._original; // The item shall only have this type if the db request for the virtual download // was successful (like: no conflicting remote remove etc). This decision is done // either in processFileAnalyzeRemoteInfo() or further down here. if (item->_type == ItemTypeVirtualFileDownload) item->_type = ItemTypeVirtualFile; if (serverEntry.isValid()) { processFileAnalyzeRemoteInfo(item, path, localEntry, serverEntry, dbEntry); return; } // Downloading a virtual file is like a server action and can happen even if // server-side nothing has changed // NOTE: Normally setting the VirtualFileDownload flag means that local and // remote will be rediscovered. This is just a fallback. if (_queryServer == ParentNotChanged && dbEntry._type == ItemTypeVirtualFileDownload) { item->_direction = SyncFileItem::Down; item->_instruction = CSYNC_INSTRUCTION_NEW; item->_type = ItemTypeVirtualFileDownload; } processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); } // Compute the checksum of the given file and assign the result in item->_checksumHeader // Returns true if the checksum was successfully computed static bool computeLocalChecksum(const QByteArray &header, const QString &path, const SyncFileItemPtr &item) { auto type = parseChecksumHeaderType(header); if (!type.isEmpty()) { // TODO: compute async? QByteArray checksum = ComputeChecksum::computeNow(path, type); if (!checksum.isEmpty()) { item->_checksumHeader = makeChecksumHeader(type, checksum); return true; } } return false; } void ProcessDirectoryJob::processFileAnalyzeRemoteInfo( const SyncFileItemPtr &item, PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry) { item->_checksumHeader = serverEntry.checksumHeader; item->_fileId = serverEntry.fileId; item->_remotePerm = serverEntry.remotePerm; item->_type = serverEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; item->_etag = serverEntry.etag; item->_previousSize = localEntry.size; item->_previousModtime = localEntry.modtime; item->_directDownloadUrl = serverEntry.directDownloadUrl; item->_directDownloadCookies = serverEntry.directDownloadCookies; // The file is known in the db already if (dbEntry.isValid()) { if (serverEntry.isDirectory != dbEntry.isDirectory()) { // If the type of the entity changed, it's like NEW, but // needs to delete the other entity first. item->_instruction = CSYNC_INSTRUCTION_TYPE_CHANGE; item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; item->_size = serverEntry.size; } else if (dbEntry._type == ItemTypeVirtualFileDownload) { item->_direction = SyncFileItem::Down; item->_instruction = CSYNC_INSTRUCTION_NEW; item->_type = ItemTypeVirtualFileDownload; } else if (dbEntry._etag != serverEntry.etag) { item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; item->_size = serverEntry.size; if (serverEntry.isDirectory) { ENFORCE(dbEntry.isDirectory()); item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; } else if (!localEntry.isValid() && _queryLocal != ParentNotChanged) { // Deleted locally, changed on server item->_instruction = CSYNC_INSTRUCTION_NEW; } else { item->_instruction = CSYNC_INSTRUCTION_SYNC; } } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; item->_direction = SyncFileItem::Down; } else { processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, ParentNotChanged); return; } processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); return; } // Unknown in db: new file on the server Q_ASSERT(!dbEntry.isValid()); item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Down; item->_modtime = serverEntry.modtime; item->_size = serverEntry.size; auto postProcessServerNew = [item, this, path, serverEntry, localEntry, dbEntry] () mutable { if (item->isDirectory()) { _pendingAsyncJobs++; _discoveryData->checkSelectiveSyncNewFolder(path._server, serverEntry.remotePerm, [=](bool result) { --_pendingAsyncJobs; if (!result) { processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); } QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); }); return; } // Turn new remote files into virtual files if the option is enabled. if (!localEntry.isValid() && _discoveryData->_syncOptions._newFilesAreVirtual && item->_type == ItemTypeFile) { item->_type = ItemTypeVirtualFile; addVirtualFileSuffix(path._original); } processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); }; // Potential NEW/NEW conflict is handled in AnalyzeLocal if (localEntry.isValid()) { postProcessServerNew(); return; } // Not in db or locally: either new or a rename Q_ASSERT(!dbEntry.isValid() && !localEntry.isValid()); // Check for renames (if there is a file with the same file id) bool done = false; bool async = false; // This function will be executed for every candidate auto renameCandidateProcessing = [&](const OCC::SyncJournalFileRecord &base) { if (done) return; if (!base.isValid()) return; if (base._type == ItemTypeVirtualFileDownload) { // Remote rename of a virtual file we have locally scheduled // for download. We just consider this NEW but mark it for download. item->_type = ItemTypeVirtualFileDownload; done = true; return; } // Some things prohibit rename detection entirely. // Since we don't do the same checks again in reconcile, we can't // just skip the candidate, but have to give up completely. if (base._type != item->_type && base._type != ItemTypeVirtualFile) { qCInfo(lcDisco, "file types different, not a rename"); done = true; return; } if (!serverEntry.isDirectory && base._etag != serverEntry.etag) { /* File with different etag, don't do a rename, but download the file again */ qCInfo(lcDisco, "file etag different, not a rename"); done = true; return; } // Now we know there is a sane rename candidate. QString originalPath = QString::fromUtf8(base._path); // Rename of a virtual file if (base._type == ItemTypeVirtualFile && item->_type == ItemTypeFile) { // Ignore if the base is a virtual files return; } if (_discoveryData->_renamedItems.contains(originalPath)) { qCInfo(lcDisco, "folder already has a rename entry, skipping"); return; } /* A remote rename can also mean Encryption Mangled Name. * if we find one of those in the database, we ignore it. */ if (!base._e2eMangledName.isEmpty()) { qCWarning(lcDisco, "Encrypted file can not rename"); done = true; return; } if (item->_type == ItemTypeFile) { csync_file_stat_t buf; if (csync_vio_local_stat((_discoveryData->_localDir + originalPath).toUtf8(), &buf)) { qCInfo(lcDisco) << "Local file does not exist anymore." << originalPath; return; } if (buf.modtime != base._modtime || buf.size != base._fileSize || buf.type != ItemTypeFile) { qCInfo(lcDisco) << "File has changed locally, not a rename." << originalPath; return; } } else { if (!QFileInfo(_discoveryData->_localDir + originalPath).isDir()) { qCInfo(lcDisco) << "Local directory does not exist anymore." << originalPath; return; } } bool wasDeletedOnServer = _discoveryData->findAndCancelDeletedJob(originalPath).first; auto postProcessRename = [this, item, base, originalPath](PathTuple &path) { auto adjustedOriginalPath = _discoveryData->adjustRenamedPath(originalPath); _discoveryData->_renamedItems.insert(originalPath, path._target); item->_modtime = base._modtime; item->_inode = base._inode; item->_instruction = CSYNC_INSTRUCTION_RENAME; item->_direction = SyncFileItem::Down; item->_renameTarget = path._target; item->_file = adjustedOriginalPath; item->_originalFile = originalPath; path._original = originalPath; path._local = adjustedOriginalPath; qCInfo(lcDisco) << "Rename detected (down) " << item->_file << " -> " << item->_renameTarget; }; if (wasDeletedOnServer) { postProcessRename(path); done = true; } else { // we need to make a request to the server to know that the original file is deleted on the server _pendingAsyncJobs++; auto job = new RequestEtagJob(_discoveryData->_account, originalPath, this); connect(job, &RequestEtagJob::finishedWithResult, this, [=](const Result &etag) mutable { _pendingAsyncJobs--; QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); if (etag.errorCode() != 404 || // Somehow another item claimed this original path, consider as if it existed _discoveryData->_renamedItems.contains(originalPath)) { // If the file exist or if there is another error, consider it is a new file. postProcessServerNew(); return; } // The file do not exist, it is a rename // In case the deleted item was discovered in parallel _discoveryData->findAndCancelDeletedJob(originalPath); postProcessRename(path); processFileFinalize(item, path, item->isDirectory(), item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist, _queryServer); }); job->start(); done = true; // Ideally, if the origin still exist on the server, we should continue searching... but that'd be difficult async = true; } }; if (!_discoveryData->_statedb->getFileRecordsByFileId(serverEntry.fileId, renameCandidateProcessing)) { dbError(); return; } if (async) { return; // We went async } if (item->_instruction == CSYNC_INSTRUCTION_NEW) { postProcessServerNew(); return; } processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); } void ProcessDirectoryJob::processFileAnalyzeLocalInfo( const SyncFileItemPtr &item, PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry, QueryMode recurseQueryServer) { bool noServerEntry = (_queryServer != ParentNotChanged && !serverEntry.isValid()) || (_queryServer == ParentNotChanged && !dbEntry.isValid()); if (noServerEntry) recurseQueryServer = ParentDontExist; bool serverModified = item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_RENAME || item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE; // Decay server modifications to UPDATE_METADATA if the local virtual exists bool hasLocalVirtual = localEntry.isVirtualFile || (_queryLocal == ParentNotChanged && dbEntry.isVirtualFile()); bool virtualFileDownload = item->_type == ItemTypeVirtualFileDownload; if (serverModified && !virtualFileDownload && hasLocalVirtual) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; serverModified = false; item->_type = ItemTypeVirtualFile; } if (dbEntry.isVirtualFile() && !virtualFileDownload) item->_type = ItemTypeVirtualFile; _childModified |= serverModified; auto finalize = [&] { bool recurse = item->isDirectory() || localEntry.isDirectory || serverEntry.isDirectory; // Even if we have a local directory: If the remote is a file that's propagated as a // conflict we don't need to recurse into it. (local c1.owncloud, c1/ ; remote: c1) if (item->_instruction == CSYNC_INSTRUCTION_CONFLICT && !item->isDirectory()) recurse = false; if (_queryLocal != NormalQuery && _queryServer != NormalQuery && !item->_isRestoration) recurse = false; auto recurseQueryLocal = _queryLocal == ParentNotChanged ? ParentNotChanged : localEntry.isDirectory || item->_instruction == CSYNC_INSTRUCTION_RENAME ? NormalQuery : ParentDontExist; processFileFinalize(item, path, recurse, recurseQueryLocal, recurseQueryServer); }; if (!localEntry.isValid()) { if (_queryLocal == ParentNotChanged && dbEntry.isValid()) { if (noServerEntry) { // Not modified locally (ParentNotChanged), but not on the server: Removed on the server. item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; } } else if (noServerEntry) { // Not locally, not on the server. The entry is stale! qCInfo(lcDisco) << "Stale DB entry"; _discoveryData->_statedb->deleteFileRecord(path._original, true); return; } else if (dbEntry._type == ItemTypeVirtualFile) { // If the virtual file is removed, recreate it. item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Down; item->_type = ItemTypeVirtualFile; } else if (!serverModified) { // Removed locally: also remove on the server. if (_dirItem && _dirItem->_isRestoration && _dirItem->_instruction == CSYNC_INSTRUCTION_NEW) { // Also restore everything item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Down; item->_isRestoration = true; item->_errorString = tr("Not allowed to remove, restoring"); } else if (!dbEntry._serverHasIgnoredFiles) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Up; } } finalize(); return; } Q_ASSERT(localEntry.isValid()); item->_inode = localEntry.inode; if (dbEntry.isValid()) { bool typeChange = localEntry.isDirectory != dbEntry.isDirectory(); if (localEntry.isVirtualFile) { if (dbEntry._type == ItemTypeFile) { // If we find what looks to be a spurious "abc.owncloud" the base file "abc" // might have been renamed to that. Make sure that the base file is not // deleted from the server. if (dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) { qCInfo(lcDisco) << "Base file was renamed to virtual file:" << item->_file; item->_direction = SyncFileItem::Down; item->_instruction = CSYNC_INSTRUCTION_NEW; item->_type = ItemTypeVirtualFile; } } if (noServerEntry) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; } } else if (!typeChange && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || localEntry.isDirectory)) { // Local file unchanged. ENFORCE(localEntry.isDirectory == dbEntry.isDirectory()); if (noServerEntry) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; } else if (!serverModified && dbEntry._inode != localEntry.inode) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; item->_direction = SyncFileItem::Down; // Does not matter } } else if (serverModified || dbEntry.isVirtualFile()) { processFileConflict(item, path, localEntry, serverEntry, dbEntry); } else if (typeChange) { item->_instruction = CSYNC_INSTRUCTION_TYPE_CHANGE; item->_direction = SyncFileItem::Up; item->_checksumHeader.clear(); item->_size = localEntry.size; item->_modtime = localEntry.modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; _childModified = true; } else { // Local file was changed item->_instruction = CSYNC_INSTRUCTION_SYNC; if (noServerEntry) { // Special case! deleted on server, modified on client, the instruction is then NEW item->_instruction = CSYNC_INSTRUCTION_NEW; } item->_direction = SyncFileItem::Up; item->_checksumHeader.clear(); item->_size = localEntry.size; item->_modtime = localEntry.modtime; item->_previousSize = dbEntry._fileSize; item->_previousModtime = dbEntry._modtime; _childModified = true; // Checksum comparison at this stage is only enabled for .eml files, // check #4754 #4755 bool isEmlFile = path._original.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive); if (isEmlFile && dbEntry._fileSize == localEntry.size && !dbEntry._checksumHeader.isEmpty()) { if (computeLocalChecksum(dbEntry._checksumHeader, _discoveryData->_localDir + path._local, item) && item->_checksumHeader == dbEntry._checksumHeader) { qCInfo(lcDisco) << "NOTE: Checksums are identical, file did not actually change: " << path._local; item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; } } } finalize(); return; } Q_ASSERT(!dbEntry.isValid()); if (localEntry.isVirtualFile && !noServerEntry) { // Somehow there is a missing DB entry while the virtual file already exists. // The instruction should already be set correctly. ASSERT(item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA); ASSERT(item->_type == ItemTypeVirtualFile); finalize(); return; } else if (serverModified) { processFileConflict(item, path, localEntry, serverEntry, dbEntry); finalize(); return; } // New local file or rename item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Up; item->_checksumHeader.clear(); item->_size = localEntry.size; item->_modtime = localEntry.modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : localEntry.isVirtualFile ? ItemTypeVirtualFile : ItemTypeFile; _childModified = true; auto postProcessLocalNew = [item, localEntry, this]() { if (localEntry.isVirtualFile) { // Remove the spurious file if it looks like a placeholder file // (we know placeholder files contain " ") if (localEntry.size <= 1) { qCWarning(lcDisco) << "Wiping virtual file without db entry for" << _currentFolder._local + "/" + localEntry.name; item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; } else { qCWarning(lcDisco) << "Virtual file without db entry for" << _currentFolder._local << localEntry.name << "but looks odd, keeping"; item->_instruction = CSYNC_INSTRUCTION_IGNORE; } } }; // Check if it is a move OCC::SyncJournalFileRecord base; if (!_discoveryData->_statedb->getFileRecordByInode(localEntry.inode, &base)) { dbError(); return; } bool isMove = base.isValid() && base._type == item->_type && ((base._modtime == localEntry.modtime && base._fileSize == localEntry.size) // Directories and virtual files don't need size/mtime equality || localEntry.isDirectory || localEntry.isVirtualFile); if (isMove) { // The old file must have been deleted. isMove = !QFile::exists(_discoveryData->_localDir + base._path); } // Verify the checksum where possible if (isMove && !base._checksumHeader.isEmpty() && item->_type == ItemTypeFile) { if (computeLocalChecksum(base._checksumHeader, _discoveryData->_localDir + path._original, item)) { qCInfo(lcDisco) << "checking checksum of potential rename " << path._original << item->_checksumHeader << base._checksumHeader; isMove = item->_checksumHeader == base._checksumHeader; } } auto originalPath = QString::fromUtf8(base._path); if (isMove && _discoveryData->_renamedItems.contains(originalPath)) isMove = false; //Check local permission if we are allowed to put move the file here // Technically we should use the one from the server, but we'll assume it is the same if (isMove && !checkMovePermissions(base._remotePerm, originalPath, item->isDirectory())) isMove = false; // Finally make it a NEW or a RENAME if (!isMove) { postProcessLocalNew(); } else { auto wasDeletedOnClient = _discoveryData->findAndCancelDeletedJob(originalPath); auto processRename = [item, originalPath, base, this](PathTuple &path) { auto adjustedOriginalPath = _discoveryData->adjustRenamedPath(originalPath); _discoveryData->_renamedItems.insert(originalPath, path._target); item->_renameTarget = path._target; path._server = adjustedOriginalPath; item->_file = path._server; path._original = originalPath; item->_originalFile = path._original; item->_modtime = base._modtime; item->_inode = base._inode; item->_instruction = CSYNC_INSTRUCTION_RENAME; item->_direction = SyncFileItem::Up; item->_fileId = base._fileId; item->_remotePerm = base._remotePerm; item->_etag = base._etag; item->_type = base._type; qCInfo(lcDisco) << "Rename detected (up) " << item->_file << " -> " << item->_renameTarget; }; if (wasDeletedOnClient.first) { recurseQueryServer = wasDeletedOnClient.second == base._etag ? ParentNotChanged : NormalQuery; processRename(path); } else { // We must query the server to know if the etag has not changed _pendingAsyncJobs++; QString serverOriginalPath = originalPath; if (base.isVirtualFile()) chopVirtualFileSuffix(serverOriginalPath); auto job = new RequestEtagJob(_discoveryData->_account, serverOriginalPath, this); connect(job, &RequestEtagJob::finishedWithResult, this, [=](const Result &etag) mutable { if (!etag || (*etag != base._etag && !item->isDirectory()) || _discoveryData->_renamedItems.contains(originalPath)) { qCInfo(lcDisco) << "Can't rename because the etag has changed or the directory is gone" << originalPath; // Can't be a rename, leave it as a new. postProcessLocalNew(); } else { // In case the deleted item was discovered in parallel _discoveryData->findAndCancelDeletedJob(originalPath); processRename(path); recurseQueryServer = *etag == base._etag ? ParentNotChanged : NormalQuery; } processFileFinalize(item, path, item->isDirectory(), NormalQuery, recurseQueryServer); _pendingAsyncJobs--; QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); }); job->start(); return; } } finalize(); } void ProcessDirectoryJob::processFileConflict(const SyncFileItemPtr &item, ProcessDirectoryJob::PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry) { item->_previousSize = localEntry.size; item->_previousModtime = localEntry.modtime; if (serverEntry.isDirectory && localEntry.isDirectory) { // Folders of the same path are always considered equals item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; return; } // A conflict with a virtual should lead to virtual file download if (dbEntry.isVirtualFile() || localEntry.isVirtualFile) item->_type = ItemTypeVirtualFileDownload; // If there's no content hash, use heuristics if (serverEntry.checksumHeader.isEmpty()) { // If the size or mtime is different, it's definitely a conflict. bool isConflict = (serverEntry.size != localEntry.size) || (serverEntry.modtime != localEntry.modtime); // It could be a conflict even if size and mtime match! // // In older client versions we always treated these cases as a // non-conflict. This behavior is preserved in case the server // doesn't provide a content checksum. // SO: If there is no checksum, we can have !isConflict here // even though the files might have different content! This is an // intentional tradeoff. Downloading and comparing files would // be technically correct in this situation but leads to too // much waste. // In particular this kind of NEW/NEW situation with identical // sizes and mtimes pops up when the local database is lost for // whatever reason. item->_instruction = isConflict ? CSYNC_INSTRUCTION_CONFLICT : CSYNC_INSTRUCTION_UPDATE_METADATA; item->_direction = isConflict ? SyncFileItem::None : SyncFileItem::Down; return; } // Do we have an UploadInfo for this? // Maybe the Upload was completed, but the connection was broken just before // we recieved the etag (Issue #5106) auto up = _discoveryData->_statedb->getUploadInfo(path._original); if (up._valid && up._contentChecksum == serverEntry.checksumHeader) { // Solve the conflict into an upload, or nothing item->_instruction = up._modtime == localEntry.modtime && up._size == localEntry.size ? CSYNC_INSTRUCTION_NONE : CSYNC_INSTRUCTION_SYNC; item->_direction = SyncFileItem::Up; // Update the etag and other server metadata in the journal already // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because // we must not store the size/modtime from the file system) OCC::SyncJournalFileRecord rec; if (_discoveryData->_statedb->getFileRecord(path._original, &rec)) { rec._path = path._original.toUtf8(); rec._etag = serverEntry.etag; rec._fileId = serverEntry.fileId; rec._modtime = serverEntry.modtime; rec._type = item->_type; rec._fileSize = serverEntry.size; rec._remotePerm = serverEntry.remotePerm; rec._checksumHeader = serverEntry.checksumHeader; _discoveryData->_statedb->setFileRecordMetadata(rec); } return; } // Rely on content hash comparisons to optimize away non-conflicts inside the job item->_instruction = CSYNC_INSTRUCTION_CONFLICT; item->_direction = SyncFileItem::None; } void ProcessDirectoryJob::processFileFinalize( const SyncFileItemPtr &item, PathTuple path, bool recurse, QueryMode recurseQueryLocal, QueryMode recurseQueryServer) { // Adjust target path for virtual-suffix files if (item->_type == ItemTypeVirtualFile) { addVirtualFileSuffix(path._target); if (item->_instruction == CSYNC_INSTRUCTION_RENAME) addVirtualFileSuffix(item->_renameTarget); else addVirtualFileSuffix(item->_file); } if (path._original != path._target && (item->_instruction == CSYNC_INSTRUCTION_UPDATE_METADATA || item->_instruction == CSYNC_INSTRUCTION_NONE)) { ASSERT(_dirItem && _dirItem->_instruction == CSYNC_INSTRUCTION_RENAME); // This is because otherwise subitems are not updated! (ideally renaming a directory could // update the database for all items! See PropagateDirectory::slotSubJobsFinished) item->_instruction = CSYNC_INSTRUCTION_RENAME; item->_renameTarget = path._target; item->_direction = _dirItem->_direction; } qCInfo(lcDisco) << "Discovered" << item->_file << csync_instruction_str(item->_instruction) << item->_direction << item->_type; if (item->isDirectory() && item->_instruction == CSYNC_INSTRUCTION_SYNC) item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; if (!checkPermissions(item)) recurse = false; if (recurse) { auto job = new ProcessDirectoryJob(item, recurseQueryLocal, recurseQueryServer, _discoveryData, this); job->_currentFolder = path; if (item->_instruction == CSYNC_INSTRUCTION_REMOVE) { job->setParent(_discoveryData); _discoveryData->_queuedDeletedDirectories[path._original] = job; } else { connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); _queuedJobs.push_back(job); } } else { if (item->_instruction == CSYNC_INSTRUCTION_REMOVE // For the purpose of rename deletion, restored deleted placeholder is as if it was deleted || (item->_type == ItemTypeVirtualFile && item->_instruction == CSYNC_INSTRUCTION_NEW)) { _discoveryData->_deletedItem[path._original] = item; } emit _discoveryData->itemDiscovered(item); } } void ProcessDirectoryJob::processBlacklisted(const PathTuple &path, const OCC::LocalInfo &localEntry, const SyncJournalFileRecord &dbEntry) { if (!localEntry.isValid()) return; auto item = SyncFileItem::fromSyncJournalFileRecord(dbEntry); item->_file = path._target; item->_originalFile = path._original; item->_inode = localEntry.inode; if (dbEntry.isValid() && ((dbEntry._modtime == localEntry.modtime && dbEntry._fileSize == localEntry.size) || (localEntry.isDirectory && dbEntry.isDirectory()))) { item->_instruction = CSYNC_INSTRUCTION_REMOVE; item->_direction = SyncFileItem::Down; } else { item->_instruction = CSYNC_INSTRUCTION_IGNORE; item->_status = SyncFileItem::FileIgnored; item->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist"); _childIgnored = true; } qCInfo(lcDisco) << "Discovered (blacklisted) " << item->_file << item->_instruction << item->_direction << item->isDirectory(); if (item->isDirectory() && item->_instruction != CSYNC_INSTRUCTION_IGNORE) { auto job = new ProcessDirectoryJob(item, NormalQuery, InBlackList, _discoveryData, this); job->_currentFolder = path; connect(job, &ProcessDirectoryJob::finished, this, &ProcessDirectoryJob::subJobFinished); _queuedJobs.push_back(job); } else { emit _discoveryData->itemDiscovered(item); } } bool ProcessDirectoryJob::checkPermissions(const OCC::SyncFileItemPtr &item) { if (item->_direction != SyncFileItem::Up) { // Currently we only check server-side permissions return true; } switch (item->_instruction) { case CSYNC_INSTRUCTION_TYPE_CHANGE: case CSYNC_INSTRUCTION_NEW: { const auto perms = !_rootPermissions.isNull() ? _rootPermissions : _dirItem ? _dirItem->_remotePerm : _rootPermissions; if (perms.isNull()) { // No permissions set return true; } else if (item->isDirectory() && !perms.hasPermission(RemotePermissions::CanAddSubDirectories)) { qCWarning(lcDisco) << "checkForPermission: ERROR" << item->_file; item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_status = SyncFileItem::NormalError; item->_errorString = tr("Not allowed because you don't have permission to add subfolders to that folder"); return false; } else if (!item->isDirectory() && !perms.hasPermission(RemotePermissions::CanAddFile)) { qCWarning(lcDisco) << "checkForPermission: ERROR" << item->_file; item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_status = SyncFileItem::NormalError; item->_errorString = tr("Not allowed because you don't have permission to add files in that folder"); return false; } break; } case CSYNC_INSTRUCTION_SYNC: { const auto perms = item->_remotePerm; if (perms.isNull()) { // No permissions set return true; } if (!perms.hasPermission(RemotePermissions::CanWrite)) { qCWarning(lcDisco) << "checkForPermission: RESTORING" << item->_file; item->_instruction = CSYNC_INSTRUCTION_CONFLICT; item->_errorString = tr("Not allowed to upload this file because it is read-only on the server, restoring"); item->_direction = SyncFileItem::Down; item->_isRestoration = true; // Take the things to write to the db from the "other" node (i.e: info from server). // Do a lookup into the csync remote tree to get the metadata we need to restore. qSwap(item->_size, item->_previousSize); qSwap(item->_modtime, item->_previousModtime); return false; } break; } case CSYNC_INSTRUCTION_REMOVE: { const auto perms = item->_remotePerm; if (perms.isNull()) { // No permissions set return true; } if (!perms.hasPermission(RemotePermissions::CanDelete)) { qCWarning(lcDisco) << "checkForPermission: RESTORING" << item->_file; item->_instruction = CSYNC_INSTRUCTION_NEW; item->_direction = SyncFileItem::Down; item->_isRestoration = true; item->_errorString = tr("Not allowed to remove, restoring"); return true; // (we need to recurse to restore sub items) } break; } default: break; } return true; } bool ProcessDirectoryJob::checkMovePermissions(RemotePermissions srcPerm, const QString &srcPath, bool isDirectory) { auto destPerms = !_rootPermissions.isNull() ? _rootPermissions : _dirItem ? _dirItem->_remotePerm : _rootPermissions; auto filePerms = srcPerm; //true when it is just a rename in the same directory. (not a move) bool isRename = srcPath.startsWith(_currentFolder._original) && srcPath.lastIndexOf('/') == _currentFolder._original.size(); // Check if we are allowed to move to the destination. bool destinationOK = true; if (isRename || destPerms.isNull()) { // no need to check for the destination dir permission destinationOK = true; } else if (isDirectory && !destPerms.hasPermission(RemotePermissions::CanAddSubDirectories)) { destinationOK = false; } else if (!isDirectory && !destPerms.hasPermission(RemotePermissions::CanAddFile)) { destinationOK = false; } // check if we are allowed to move from the source bool sourceOK = true; if (!filePerms.isNull() && ((isRename && !filePerms.hasPermission(RemotePermissions::CanRename)) || (!isRename && !filePerms.hasPermission(RemotePermissions::CanMove)))) { // We are not allowed to move or rename this file sourceOK = false; } if (!sourceOK || !destinationOK) { qCInfo(lcDisco) << "Not a move because permission does not allow it." << sourceOK << destinationOK; if (!sourceOK) { // This is the behavior that we had in the client <= 2.5. // but that might not be needed anymore _discoveryData->_statedb->avoidRenamesOnNextSync(srcPath); } return false; } return true; } void ProcessDirectoryJob::subJobFinished() { auto job = qobject_cast(sender()); ASSERT(job); _childIgnored |= job->_childIgnored; _childModified |= job->_childModified; if (job->_dirItem) emit _discoveryData->itemDiscovered(job->_dirItem); int count = _runningJobs.removeAll(job); ASSERT(count == 1); job->deleteLater(); QTimer::singleShot(0, _discoveryData, &DiscoveryPhase::scheduleMoreJobs); } int ProcessDirectoryJob::processSubJobs(int nbJobs) { if (_queuedJobs.empty() && _runningJobs.empty() && _pendingAsyncJobs == 0) { _pendingAsyncJobs = -1; // We're finished, we don't want to emit finished again if (_dirItem) { if (_childModified && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { // re-create directory that has modified contents _dirItem->_instruction = CSYNC_INSTRUCTION_NEW; } if (_childModified && _dirItem->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE && !_dirItem->isDirectory()) { // Replacing a directory by a file is a conflict, if the directory had modified children _dirItem->_instruction = CSYNC_INSTRUCTION_CONFLICT; if (_dirItem->_direction == SyncFileItem::Up) { _dirItem->_type = ItemTypeDirectory; _dirItem->_direction = SyncFileItem::Down; } } if (_childIgnored && _dirItem->_instruction == CSYNC_INSTRUCTION_REMOVE) { // Do not remove a directory that has ignored files _dirItem->_instruction = CSYNC_INSTRUCTION_NONE; } } emit finished(); } int started = 0; foreach (auto *rj, _runningJobs) { started += rj->processSubJobs(nbJobs - started); if (started >= nbJobs) return started; } while (started < nbJobs && !_queuedJobs.empty()) { auto f = _queuedJobs.front(); _queuedJobs.pop_front(); _runningJobs.push_back(f); f->start(); started++; } return started; } void ProcessDirectoryJob::dbError() { _discoveryData->fatalError(tr("Error while reading the database")); _pendingAsyncJobs = -1; // We're finished, we don't want to emit finished again emit finished(); } void ProcessDirectoryJob::addVirtualFileSuffix(QString &str) const { str.append(_discoveryData->_syncOptions._virtualFileSuffix); } bool ProcessDirectoryJob::hasVirtualFileSuffix(const QString &str) const { return str.endsWith(_discoveryData->_syncOptions._virtualFileSuffix); } void ProcessDirectoryJob::chopVirtualFileSuffix(QString &str) const { bool hasSuffix = hasVirtualFileSuffix(str); ASSERT(hasSuffix); if (hasSuffix) str.chop(_discoveryData->_syncOptions._virtualFileSuffix.size()); } DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() { auto serverJob = new DiscoverySingleDirectoryJob(_discoveryData->_account, _discoveryData->_remoteFolder + _currentFolder._server, this); connect(serverJob, &DiscoverySingleDirectoryJob::etag, this, &ProcessDirectoryJob::etag); _discoveryData->_currentlyActiveJobs++; _pendingAsyncJobs++; connect(serverJob, &DiscoverySingleDirectoryJob::finished, this, [this, serverJob](const auto &results) { _discoveryData->_currentlyActiveJobs--; _pendingAsyncJobs--; if (results) { _serverNormalQueryEntries = *results; _serverQueryDone = true; if (!serverJob->_dataFingerprint.isEmpty() && _discoveryData->_dataFingerprint.isEmpty()) _discoveryData->_dataFingerprint = serverJob->_dataFingerprint; if (_localQueryDone) process(); } else { if (results.errorCode() == 403) { // 403 Forbidden can be sent by the server if the file firewall is active. // A file or directory should be ignored and sync must continue. See #3490 qCWarning(lcDisco, "Directory access Forbidden (File Firewall?)"); if (_dirItem) { _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE; _dirItem->_errorString = results.errorMessage(); emit finished(); return; } } else if (results.errorCode() == 503) { // The server usually replies with the custom "503 Storage not available" // if some path is temporarily unavailable. But in some cases a standard 503 // is returned too. Thus we can't distinguish the two and will treat any // 503 as request to ignore the folder. See #3113 #2884. qCWarning(lcDisco(), "Storage was not available!"); if (_dirItem) { _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE; _dirItem->_errorString = results.errorMessage(); emit finished(); return; } } emit _discoveryData->fatalError(tr("Server replied with an error while reading directory '%1' : %2") .arg(_currentFolder._server, results.errorMessage())); emit finished(); } }); connect(serverJob, &DiscoverySingleDirectoryJob::firstDirectoryPermissions, this, [this](const RemotePermissions &perms) { _rootPermissions = perms; }); serverJob->start(); return serverJob; } bool ProcessDirectoryJob::runLocalQuery() { auto dh = csync_vio_local_opendir((_discoveryData->_localDir + _currentFolder._local).toUtf8()); if (!dh) { qCInfo(lcDisco) << "Error while opening directory" << (_discoveryData->_localDir + _currentFolder._local) << errno; QString errorString = tr("Error while opening directory %1").arg(_discoveryData->_localDir + _currentFolder._local); if (errno == EACCES) { errorString = tr("Directory not accessible on client, permission denied"); if (_dirItem) { _dirItem->_instruction = CSYNC_INSTRUCTION_IGNORE; _dirItem->_errorString = errorString; emit finished(); return false; } } else if (errno == ENOENT) { errorString = tr("Directory not found: %1").arg(_discoveryData->_localDir + _currentFolder._local); } else if (errno == ENOTDIR) { // Not a directory.. // Just consider it is empty return true; } emit _discoveryData->fatalError(errorString); emit finished(); return false; } errno = 0; while (auto dirent = csync_vio_local_readdir(dh)) { if (dirent->type == ItemTypeSkip) continue; LocalInfo i; static QTextCodec *codec = QTextCodec::codecForName("UTF-8"); ASSERT(codec); QTextCodec::ConverterState state; i.name = codec->toUnicode(dirent->path, dirent->path.size(), &state); if (state.invalidChars > 0 || state.remainingChars > 0) { _childIgnored = true; auto item = SyncFileItemPtr::create(); item->_file = _currentFolder._target + i.name; item->_instruction = CSYNC_INSTRUCTION_IGNORE; item->_status = SyncFileItem::NormalError; item->_errorString = tr("Filename encoding is not valid"); emit _discoveryData->itemDiscovered(item); continue; } i.modtime = dirent->modtime; i.size = dirent->size; i.inode = dirent->inode; i.isDirectory = dirent->type == ItemTypeDirectory; i.isHidden = dirent->is_hidden; i.isSymLink = dirent->type == ItemTypeSoftLink; _localNormalQueryEntries.push_back(i); } csync_vio_local_closedir(dh); if (errno != 0) { // Note: Windows vio converts any error into EACCES qCWarning(lcDisco) << "readdir failed for file in " << _currentFolder._local << " - errno: " << errno; emit _discoveryData->fatalError(tr("Error while reading directory %1").arg(_discoveryData->_localDir + _currentFolder._local)); emit finished(); return false; } return true; } }