From e0abbc0b836ee117e42564595feb0cb2ff04777f Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 8 Jul 2015 17:14:05 +0200 Subject: [PATCH 01/19] vio_local: Properly detect if files are hidden. On Linux and Mac, check for the leading dot. On Windows, there is an flag set in the result struct of fetFileInformationByHandle. --- csync/src/vio/csync_vio_local_win.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/csync/src/vio/csync_vio_local_win.c b/csync/src/vio/csync_vio_local_win.c index de08441444..8f55356ab2 100644 --- a/csync/src/vio/csync_vio_local_win.c +++ b/csync/src/vio/csync_vio_local_win.c @@ -224,7 +224,12 @@ int csync_vio_local_stat(const char *uri, csync_vio_file_stat_t *buf) { buf->type = CSYNC_VIO_FILE_TYPE_REGULAR; break; } while (0); - /* TODO Do we want to parse for CSYNC_VIO_FILE_FLAGS_HIDDEN ? */ + + /* Check for the hidden flag */ + if( fileInfo.dwFileAttributes & FILE_ATTRIBUTE_HIDDEN ) { + buf->flags |= CSYNC_VIO_FILE_FLAGS_HIDDEN; + } + buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_FLAGS; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE; From 7aa2b5082870f808952dabd3799e48fa85885dbb Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 8 Jul 2015 18:06:38 +0200 Subject: [PATCH 02/19] csync_update: Handling hidden files as excluded files, as intended. --- csync/src/csync.h | 3 ++- csync/src/csync_exclude.h | 3 ++- csync/src/csync_update.c | 42 ++++++++++++++++++++------------------ src/libsync/syncengine.cpp | 3 +++ 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/csync/src/csync.h b/csync/src/csync.h index d710c170c9..97eb9811f2 100644 --- a/csync/src/csync.h +++ b/csync/src/csync.h @@ -103,7 +103,8 @@ enum csync_status_codes_e { CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST, CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS, CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME, - CYSNC_STATUS_FILE_LOCKED_OR_OPEN + CYSNC_STATUS_FILE_LOCKED_OR_OPEN, + CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN }; typedef enum csync_status_codes_e CSYNC_STATUS; diff --git a/csync/src/csync_exclude.h b/csync/src/csync_exclude.h index 329d95763c..0bdbb6f74f 100644 --- a/csync/src/csync_exclude.h +++ b/csync/src/csync_exclude.h @@ -27,7 +27,8 @@ enum csync_exclude_type_e { CSYNC_FILE_EXCLUDE_AND_REMOVE, CSYNC_FILE_EXCLUDE_LIST, CSYNC_FILE_EXCLUDE_INVALID_CHAR, - CSYNC_FILE_EXCLUDE_LONG_FILENAME + CSYNC_FILE_EXCLUDE_LONG_FILENAME, + CSYNC_FILE_EXCLUDE_HIDDEN }; typedef enum csync_exclude_type_e CSYNC_EXCLUDE_TYPE; diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index 4274467b57..2db4b28c4b 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -163,29 +163,29 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, len = strlen(path); - /* This code should probably be in csync_exclude, but it does not have the fs parameter. - Keep it here for now and TODO also find out if we want this for Windows - https://github.com/owncloud/mirall/issues/2086 */ - if (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); - return 0; - } - /* Check if file is excluded */ excluded = csync_excluded(ctx, path,type); if (excluded != CSYNC_NOT_EXCLUDED) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded); - if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) { - return 1; - } - if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { - return 1; - } + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded); + if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) { + return 1; + } + if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { + return 1; + } - if (ctx->current_fs) { - ctx->current_fs->has_ignored_files = true; - } + if (ctx->current_fs) { + ctx->current_fs->has_ignored_files = true; + } + } else { + + /* This code should probably be in csync_exclude, but it does not have the fs parameter. + Keep it here for now */ + if (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); + excluded = CSYNC_FILE_EXCLUDE_HIDDEN; + } } if (ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncBlackListHook) { @@ -208,9 +208,9 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, st->child_modified = 0; st->has_ignored_files = 0; - /* check hardlink count */ + /* FIXME: Under which conditions are the following two ifs true and the code + * is executed? */ if (type == CSYNC_FTW_TYPE_FILE ) { - if (fs->mtime == 0) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path); @@ -424,6 +424,8 @@ out: st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */ } else if (excluded == CSYNC_FILE_EXCLUDE_LONG_FILENAME) { st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME; /* File name is too long. */ + } else if (excluded == CSYNC_FILE_EXCLUDE_HIDDEN ) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN; } } if (st->instruction != CSYNC_INSTRUCTION_NONE && st->instruction != CSYNC_INSTRUCTION_IGNORE diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index dba2a47d23..2ceb6c15cb 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -376,6 +376,9 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME: item->_errorString = tr("Filename is too long."); break; + case CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN: + item->_errorString = tr("File is ignored because it's hidden."); + break; case CYSNC_STATUS_FILE_LOCKED_OR_OPEN: item->_errorString = QLatin1String("File locked"); // don't translate, internal use! break; From c940762b7ceedc9f1851748a7051735aa873c01d Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Thu, 9 Jul 2015 10:44:12 +0200 Subject: [PATCH 03/19] csync_update: set has_ignored_files in case of hidden files. Re-arragned the code a bit for better overview on whats happening. --- csync/src/csync_update.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index 2db4b28c4b..5dbe27bcf2 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -174,12 +174,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { return 1; } - - if (ctx->current_fs) { - ctx->current_fs->has_ignored_files = true; - } } else { - /* This code should probably be in csync_exclude, but it does not have the fs parameter. Keep it here for now */ if (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN) { @@ -242,12 +237,14 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, st->instruction = CSYNC_INSTRUCTION_NONE; goto out; } + if (excluded > CSYNC_NOT_EXCLUDED || type == CSYNC_FTW_TYPE_SLINK) { - if( type == CSYNC_FTW_TYPE_SLINK ) { - st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */ - } st->instruction = CSYNC_INSTRUCTION_IGNORE; - goto out; + if (ctx->current_fs) { + ctx->current_fs->has_ignored_files = true; + } + + goto out; } /* Update detection: Check if a database entry exists. @@ -418,15 +415,19 @@ out: /* Set the ignored error string. */ if (st->instruction == CSYNC_INSTRUCTION_IGNORE) { - if (excluded == CSYNC_FILE_EXCLUDE_LIST) { - st->error_status = CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST; /* File listed on ignore list. */ - } else if (excluded == CSYNC_FILE_EXCLUDE_INVALID_CHAR) { - st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */ - } else if (excluded == CSYNC_FILE_EXCLUDE_LONG_FILENAME) { - st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME; /* File name is too long. */ - } else if (excluded == CSYNC_FILE_EXCLUDE_HIDDEN ) { - st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN; - } + if( type == CSYNC_FTW_TYPE_SLINK ) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_SYMLINK; /* Symbolic links are ignored. */ + } else { + if (excluded == CSYNC_FILE_EXCLUDE_LIST) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_IGNORE_LIST; /* File listed on ignore list. */ + } else if (excluded == CSYNC_FILE_EXCLUDE_INVALID_CHAR) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_IS_INVALID_CHARS; /* File contains invalid characters. */ + } else if (excluded == CSYNC_FILE_EXCLUDE_LONG_FILENAME) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_LONG_FILENAME; /* File name is too long. */ + } else if (excluded == CSYNC_FILE_EXCLUDE_HIDDEN ) { + st->error_status = CSYNC_STATUS_INDIVIDUAL_EXCLUDE_HIDDEN; + } + } } if (st->instruction != CSYNC_INSTRUCTION_NONE && st->instruction != CSYNC_INSTRUCTION_IGNORE && type != CSYNC_FTW_TYPE_DIR) { From da87a47eaf5ef3f5c867c3f3175a84f06f75863f Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Thu, 9 Jul 2015 12:01:16 +0200 Subject: [PATCH 04/19] discovery: handle dot files on server as hidden. --- src/libsync/discoveryphase.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 8048b83550..60243060e9 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -311,6 +311,9 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(QString file,QMap if (!file_stat->etag || strlen(file_stat->etag) == 0) { qDebug() << "WARNING: etag of" << file_stat->name << "is" << file_stat->etag << " This must not happen."; } + if( file.startsWith(QChar('.')) ) { + file_stat->flags = CSYNC_VIO_FILE_FLAGS_HIDDEN; + } //qDebug() << "!!!!" << file_stat << file_stat->name << file_stat->file_id << map.count(); _results.append(file_stat); } From b0aedef4b01d63114e168f661b0c6f760c17c33c Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Thu, 9 Jul 2015 15:57:56 +0200 Subject: [PATCH 05/19] Sync: Store information about ignored files on the server in database. That allows better handling of remove requests that should not be done because the directory is not empty on the remote side (as it contains ignored files). --- csync/src/csync.c | 1 + csync/src/csync.h | 3 +++ csync/src/csync_statedb.c | 5 ++++- src/libsync/filesystem.cpp | 3 +++ src/libsync/syncengine.cpp | 5 ++++- src/libsync/syncfileitem.h | 3 ++- src/libsync/syncjournaldb.cpp | 20 ++++++++++++++++---- src/libsync/syncjournalfilerecord.cpp | 4 ++-- src/libsync/syncjournalfilerecord.h | 11 ++++++----- 9 files changed, 41 insertions(+), 14 deletions(-) diff --git a/csync/src/csync.c b/csync/src/csync.c index 095809a026..985e084646 100644 --- a/csync/src/csync.c +++ b/csync/src/csync.c @@ -435,6 +435,7 @@ static int _csync_treewalk_visitor(void *obj, void *data) { trav.error_status = cur->error_status; trav.should_update_metadata = cur->should_update_metadata; + trav.has_ignored_files = cur->has_ignored_files; if( other_node ) { csync_file_stat_t *other_stat = (csync_file_stat_t*)other_node->data; diff --git a/csync/src/csync.h b/csync/src/csync.h index 97eb9811f2..99e3695d0a 100644 --- a/csync/src/csync.h +++ b/csync/src/csync.h @@ -250,6 +250,9 @@ struct csync_tree_walk_file_s { /* For directories: If the etag has been updated and need to be writen on the db */ int should_update_metadata; + /* For directories: Does it have children that were ignored (hidden or ignore pattern) */ + int has_ignored_files; + const char *rename_path; const char *etag; const char *file_id; diff --git a/csync/src/csync_statedb.c b/csync/src/csync_statedb.c index 779bb34046..c951fe90f0 100644 --- a/csync/src/csync_statedb.c +++ b/csync/src/csync_statedb.c @@ -283,6 +283,9 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3 if(column_count > 12 && sqlite3_column_int64(stmt,12)) { (*st)->size = sqlite3_column_int64(stmt, 12); } + if(column_count > 13) { + (*st)->has_ignored_files = sqlite3_column_int(stmt, 13); + } } } else { if( rc != SQLITE_DONE ) { @@ -435,7 +438,7 @@ char *csync_statedb_get_etag( CSYNC *ctx, uint64_t jHash ) { return ret; } -#define BELOW_PATH_QUERY "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize FROM metadata WHERE pathlen>? AND path LIKE(?)" +#define BELOW_PATH_QUERY "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE pathlen>? AND path LIKE(?)" int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) { int rc; diff --git a/src/libsync/filesystem.cpp b/src/libsync/filesystem.cpp index 62484fc8a5..f299d1f93d 100644 --- a/src/libsync/filesystem.cpp +++ b/src/libsync/filesystem.cpp @@ -105,6 +105,9 @@ void FileSystem::setFileHidden(const QString& filename, bool hidden) SetFileAttributesW((wchar_t*)fName.utf16(), dwAttrs & ~FILE_ATTRIBUTE_HIDDEN ); } } +#else + Q_UNUSED(filename); + Q_UNUSED(hidden); #endif } diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 2ceb6c15cb..53fb7dfee4 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -350,6 +350,9 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) } item->_should_update_metadata = item->_should_update_metadata || file->should_update_metadata; + item->_hasIgnoredFiles = file->has_ignored_files > 0 ? true : false; + qDebug() << "HAS IGNORED FILES for " << item->_file << item->_hasIgnoredFiles << key; + // record the seen files to be able to clean the journal later _seenFiles.insert(item->_file); if (!renameTarget.isEmpty()) { @@ -426,7 +429,7 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) item->_type = SyncFileItem::UnknownType; } - SyncFileItem::Direction dir; + SyncFileItem::Direction dir = SyncFileItem::None; int re = 0; switch(file->instruction) { diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 0f4c401eee..8c2b933a3d 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -64,7 +64,7 @@ public: }; SyncFileItem() : _type(UnknownType), _direction(None), _isDirectory(false), - _hasBlacklistEntry(false), _status(NoStatus), + _hasIgnoredFiles(false), _hasBlacklistEntry(false), _status(NoStatus), _isRestoration(false), _should_update_metadata(false), _httpErrorCode(0), _requestDuration(0), _affectedItems(1), _instruction(CSYNC_INSTRUCTION_NONE), _modtime(0), _size(0), _inode(0) @@ -130,6 +130,7 @@ public: Type _type BITFIELD(3); Direction _direction BITFIELD(2); bool _isDirectory BITFIELD(1); + bool _hasIgnoredFiles BITFIELD(1); /// Whether there's an entry in the blacklist table. /// Note: that entry may have retries left, so this can be true diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp index f75fdb9201..4801c86056 100644 --- a/src/libsync/syncjournaldb.cpp +++ b/src/libsync/syncjournaldb.cpp @@ -346,13 +346,13 @@ bool SyncJournalDb::checkConnect() } _getFileRecordQuery.reset(new SqlQuery(_db)); - _getFileRecordQuery->prepare("SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize FROM " + _getFileRecordQuery->prepare("SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM " "metadata WHERE phash=?1" ); _setFileRecordQuery.reset(new SqlQuery(_db) ); _setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " - "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize) " - "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13);" ); + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote) " + "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14);" ); _getDownloadInfoQuery.reset(new SqlQuery(_db) ); _getDownloadInfoQuery->prepare( "SELECT tmpfile, etag, errorcount FROM " @@ -516,6 +516,16 @@ bool SyncJournalDb::updateMetadataTableStructure() commitInternal("update database structure: add pathlen index"); } + + if( columns.indexOf(QLatin1String("ignoredChildrenRemote")) == -1 ) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN ignoredChildrenRemote INT;"); + if( !query.exec()) { + sqlFail("updateMetadataTableStructure: add ignoredChildrenRemote column", query); + re = false; + } + commitInternal("update database structure: add ignoredChildrenRemote col"); + } return re; } @@ -630,6 +640,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) _setFileRecordQuery->bindValue(11, fileId ); _setFileRecordQuery->bindValue(12, remotePerm ); _setFileRecordQuery->bindValue(13, record._fileSize ); + _setFileRecordQuery->bindValue(14, record._hasIgnoredFiles ? 1:0); if( !_setFileRecordQuery->exec() ) { qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() << " :" @@ -640,7 +651,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) qDebug() << _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode << record._mode << QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type) - << record._etag << record._fileId << record._remotePerm << record._fileSize; + << record._etag << record._fileId << record._remotePerm << record._fileSize << (record._hasIgnoredFiles ? 1:0); _setFileRecordQuery->reset(); return true; @@ -719,6 +730,7 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename ) rec._fileId = _getFileRecordQuery->baValue(8); rec._remotePerm = _getFileRecordQuery->baValue(9); rec._fileSize = _getFileRecordQuery->int64Value(10); + rec._hasIgnoredFiles = _getFileRecordQuery->intValue(11) > 0 ? true : false; } else { QString err = _getFileRecordQuery->error(); qDebug() << "No journal entry found for " << filename; diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp index 01cefe4001..8bca2986e1 100644 --- a/src/libsync/syncjournalfilerecord.cpp +++ b/src/libsync/syncjournalfilerecord.cpp @@ -28,14 +28,14 @@ namespace OCC { SyncJournalFileRecord::SyncJournalFileRecord() - :_inode(0), _type(0), _fileSize(0), _mode(0) + :_inode(0), _type(0), _fileSize(0), _mode(0), _hasIgnoredFiles(false) { } SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QString &localFileName) : _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)), _type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size), - _remotePerm(item._remotePerm), _mode(0) + _remotePerm(item._remotePerm), _mode(0), _hasIgnoredFiles(item._hasIgnoredFiles) { // use the "old" inode coming with the item for the case where the // filesystem stat fails. That can happen if the the file was removed diff --git a/src/libsync/syncjournalfilerecord.h b/src/libsync/syncjournalfilerecord.h index be5f7d6acb..b18a2abf7e 100644 --- a/src/libsync/syncjournalfilerecord.h +++ b/src/libsync/syncjournalfilerecord.h @@ -37,15 +37,16 @@ public: return !_path.isEmpty(); } - QString _path; - quint64 _inode; - QDateTime _modtime; - int _type; + QString _path; + quint64 _inode; + QDateTime _modtime; + int _type; QByteArray _etag; QByteArray _fileId; qint64 _fileSize; QByteArray _remotePerm; - int _mode; + int _mode; + bool _hasIgnoredFiles; }; bool OWNCLOUDSYNC_EXPORT From e11f59af3277a9dbd93077850a302ef9d7a46acd Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Fri, 10 Jul 2015 16:48:01 +0200 Subject: [PATCH 06/19] csync_update: store the remote ignore information into tree (from tmp) This way the propagator is able to skip a remove if ingored files are exiting. --- csync/src/csync_update.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index 5dbe27bcf2..a848ff8eb0 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -265,10 +265,10 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, /* we have an update! */ CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Database entry found, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64 - ", size: %" PRId64 " <-> %" PRId64 ", perms: %s <-> %s", + ", size: %" PRId64 " <-> %" PRId64 ", perms: %s <-> %s, ignore: %d", ((int64_t) fs->mtime), ((int64_t) tmp->modtime), fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode, - (uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm ); + (uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files ); if( !fs->etag) { st->instruction = CSYNC_INSTRUCTION_EVAL; goto out; @@ -313,6 +313,12 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Need to update metadata for: %s", path); st->should_update_metadata = true; } + /* If it was remembered in the db that the remote dir has ignored files, store + * that so that the reconciler can make advantage of. + */ + if( ctx->current == REMOTE_REPLICA ) { + st->has_ignored_files = tmp->has_ignored_files; + } st->instruction = CSYNC_INSTRUCTION_NONE; } else { enum csync_vio_file_type_e tmp_vio_type = CSYNC_VIO_FILE_TYPE_UNKNOWN; From 7b8dc9777ed35bc5132c3ccac7bec55dc32daf6a Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Mon, 13 Jul 2015 15:41:11 +0200 Subject: [PATCH 07/19] Sync: Rename ignored file member to _serverHasIgnoredFiles. This should make it more obvious that only the server state about ignored files is stored in the db. --- src/libsync/syncfileitem.h | 4 ++-- src/libsync/syncjournaldb.cpp | 6 +++--- src/libsync/syncjournalfilerecord.cpp | 4 ++-- src/libsync/syncjournalfilerecord.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 8c2b933a3d..80ba367555 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -64,7 +64,7 @@ public: }; SyncFileItem() : _type(UnknownType), _direction(None), _isDirectory(false), - _hasIgnoredFiles(false), _hasBlacklistEntry(false), _status(NoStatus), + _serverHasIgnoredFiles(false), _hasBlacklistEntry(false), _status(NoStatus), _isRestoration(false), _should_update_metadata(false), _httpErrorCode(0), _requestDuration(0), _affectedItems(1), _instruction(CSYNC_INSTRUCTION_NONE), _modtime(0), _size(0), _inode(0) @@ -130,7 +130,7 @@ public: Type _type BITFIELD(3); Direction _direction BITFIELD(2); bool _isDirectory BITFIELD(1); - bool _hasIgnoredFiles BITFIELD(1); + bool _serverHasIgnoredFiles BITFIELD(1); /// Whether there's an entry in the blacklist table. /// Note: that entry may have retries left, so this can be true diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp index 4801c86056..e9dd5a5f09 100644 --- a/src/libsync/syncjournaldb.cpp +++ b/src/libsync/syncjournaldb.cpp @@ -640,7 +640,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) _setFileRecordQuery->bindValue(11, fileId ); _setFileRecordQuery->bindValue(12, remotePerm ); _setFileRecordQuery->bindValue(13, record._fileSize ); - _setFileRecordQuery->bindValue(14, record._hasIgnoredFiles ? 1:0); + _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1:0); if( !_setFileRecordQuery->exec() ) { qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() << " :" @@ -651,7 +651,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record ) qDebug() << _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode << record._mode << QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type) - << record._etag << record._fileId << record._remotePerm << record._fileSize << (record._hasIgnoredFiles ? 1:0); + << record._etag << record._fileId << record._remotePerm << record._fileSize << (record._serverHasIgnoredFiles ? 1:0); _setFileRecordQuery->reset(); return true; @@ -730,7 +730,7 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename ) rec._fileId = _getFileRecordQuery->baValue(8); rec._remotePerm = _getFileRecordQuery->baValue(9); rec._fileSize = _getFileRecordQuery->int64Value(10); - rec._hasIgnoredFiles = _getFileRecordQuery->intValue(11) > 0 ? true : false; + rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0); } else { QString err = _getFileRecordQuery->error(); qDebug() << "No journal entry found for " << filename; diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp index 8bca2986e1..cdc808a727 100644 --- a/src/libsync/syncjournalfilerecord.cpp +++ b/src/libsync/syncjournalfilerecord.cpp @@ -28,14 +28,14 @@ namespace OCC { SyncJournalFileRecord::SyncJournalFileRecord() - :_inode(0), _type(0), _fileSize(0), _mode(0), _hasIgnoredFiles(false) + :_inode(0), _type(0), _fileSize(0), _mode(0), _serverHasIgnoredFiles(false) { } SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QString &localFileName) : _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)), _type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size), - _remotePerm(item._remotePerm), _mode(0), _hasIgnoredFiles(item._hasIgnoredFiles) + _remotePerm(item._remotePerm), _mode(0), _serverHasIgnoredFiles(item._serverHasIgnoredFiles) { // use the "old" inode coming with the item for the case where the // filesystem stat fails. That can happen if the the file was removed diff --git a/src/libsync/syncjournalfilerecord.h b/src/libsync/syncjournalfilerecord.h index b18a2abf7e..f3546c5367 100644 --- a/src/libsync/syncjournalfilerecord.h +++ b/src/libsync/syncjournalfilerecord.h @@ -46,7 +46,7 @@ public: qint64 _fileSize; QByteArray _remotePerm; int _mode; - bool _hasIgnoredFiles; + bool _serverHasIgnoredFiles; }; bool OWNCLOUDSYNC_EXPORT From 9461e4ccd1fd33199c02c76353fec69518e859e4 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Mon, 13 Jul 2015 15:42:16 +0200 Subject: [PATCH 08/19] Sync: Only copy the hasIgnoredFlag for the remote side. Also added some documentation about the handling. --- src/libsync/syncengine.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 53fb7dfee4..76fda752d2 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -348,10 +348,25 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) if (file->remotePerm && file->remotePerm[0]) { item->_remotePerm = QByteArray(file->remotePerm); } + item->_should_update_metadata = item->_should_update_metadata || file->should_update_metadata; - item->_hasIgnoredFiles = file->has_ignored_files > 0 ? true : false; - qDebug() << "HAS IGNORED FILES for " << item->_file << item->_hasIgnoredFiles << key; + /* The flag "serverHasIgnoredFiles" is true if item in question is a directory + * that has children which are ignored in sync, either because the files are + * matched by an ignore pattern, or because they are hidden. + * + * Only the information about the server side ignored files is stored to the + * database and thus written to the item here. For the local repository its + * generated by the walk through the real file tree by discovery phase. + * + * It needs to go to the sync journal becasue the stat information about remote + * files are often read from database rather than being pulled from remote. + */ + if( remote ) { + item->_serverHasIgnoredFiles = (file->has_ignored_files > 0); + } + + qDebug() << "HAS IGNORED FILES for " << item->_file << item->_serverHasIgnoredFiles << key; // record the seen files to be able to clean the journal later _seenFiles.insert(item->_file); From 3a5f66e976f26f027dff33a55c7c5e07d20d7310 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Mon, 13 Jul 2015 15:46:19 +0200 Subject: [PATCH 09/19] SyncEngine: Remove bogus logging. --- src/libsync/syncengine.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 76fda752d2..2db1530990 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -366,8 +366,6 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote ) item->_serverHasIgnoredFiles = (file->has_ignored_files > 0); } - qDebug() << "HAS IGNORED FILES for " << item->_file << item->_serverHasIgnoredFiles << key; - // record the seen files to be able to clean the journal later _seenFiles.insert(item->_file); if (!renameTarget.isEmpty()) { From a4336092f61fe87b372f6853d2549c097c8f3860 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Mon, 13 Jul 2015 22:15:02 +0200 Subject: [PATCH 10/19] Hidden Files: Add a setting to ignore hidden files or not. This setting goes to the folder definition. By default, no hidden files are synced. --- csync/src/csync.c | 1 + csync/src/csync_private.h | 2 ++ csync/src/csync_update.c | 2 +- src/gui/folder.cpp | 5 +++++ src/gui/folder.h | 2 ++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/csync/src/csync.c b/csync/src/csync.c index 985e084646..0122b5f29b 100644 --- a/csync/src/csync.c +++ b/csync/src/csync.c @@ -593,6 +593,7 @@ int csync_commit(CSYNC *ctx) { ctx->remote.read_from_db = 0; ctx->read_remote_from_db = true; ctx->db_is_empty = false; + ctx->ignore_hidden_files = true; // do NOT sync hidden files by default. /* Create new trees */ diff --git a/csync/src/csync_private.h b/csync/src/csync_private.h index 973437b59b..4209bbe8ad 100644 --- a/csync/src/csync_private.h +++ b/csync/src/csync_private.h @@ -168,6 +168,8 @@ struct csync_s { */ bool db_is_empty; + bool ignore_hidden_files; + struct csync_owncloud_ctx_s *owncloud_context; }; diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index a848ff8eb0..3fd98eb412 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -177,7 +177,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, } else { /* This code should probably be in csync_exclude, but it does not have the fs parameter. Keep it here for now */ - if (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN) { + if (ctx->ignore_hidden_files && (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN)) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); excluded = CSYNC_FILE_EXCLUDE_HIDDEN; } diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index b2cc62ffb9..f83ad78834 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -813,6 +813,9 @@ void Folder::startSync(const QStringList &pathList) return; } + // pass the setting if hidden files are to be ignored, will be read in csync_update + _csync_ctx->ignore_hidden_files = _definition.ignoreHiddenFiles; + _engine.reset(new SyncEngine( _accountState->account(), _csync_ctx, path(), remoteUrl().path(), remotePath(), &_journal)); qRegisterMetaType("SyncFileItemVector"); @@ -1086,6 +1089,7 @@ void FolderDefinition::save(QSettings& settings, const FolderDefinition& folder) settings.setValue(QLatin1String("localPath"), folder.localPath); settings.setValue(QLatin1String("targetPath"), folder.targetPath); settings.setValue(QLatin1String("paused"), folder.paused); + settings.setValue(QLatin1String("ignoreHiddenFiles"), folder.ignoreHiddenFiles); settings.endGroup(); } @@ -1097,6 +1101,7 @@ bool FolderDefinition::load(QSettings& settings, const QString& alias, folder->localPath = settings.value(QLatin1String("localPath")).toString(); folder->targetPath = settings.value(QLatin1String("targetPath")).toString(); folder->paused = settings.value(QLatin1String("paused")).toBool(); + folder->ignoreHiddenFiles = settings.value(QLatin1String("ignoreHiddenFiles"), QVariant(true)).toBool(); settings.endGroup(); return true; } diff --git a/src/gui/folder.h b/src/gui/folder.h index 72843ed7cc..837cae4209 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -63,6 +63,8 @@ public: QString targetPath; /// whether the folder is paused bool paused; + /// whether the folder syncs hidden files + bool ignoreHiddenFiles; /// Saves the folder definition, creating a new settings group. static void save(QSettings& settings, const FolderDefinition& folder); From 711ae1d347f2d9795511e1d34c363bc1e17f10da Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Tue, 14 Jul 2015 12:00:22 +0200 Subject: [PATCH 11/19] FolderWatcher: Add flag to ignore hidden files (or not). --- src/gui/folderwatcher.cpp | 25 ++++++++++++++++++++----- src/gui/folderwatcher.h | 5 +++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/gui/folderwatcher.cpp b/src/gui/folderwatcher.cpp index 874203982f..b76480f704 100644 --- a/src/gui/folderwatcher.cpp +++ b/src/gui/folderwatcher.cpp @@ -36,7 +36,8 @@ namespace OCC { FolderWatcher::FolderWatcher(const QString &root, QObject *parent) - : QObject(parent) + : QObject(parent), + _ignoreHidden(true) { _d.reset(new FolderWatcherPrivate(this, root)); @@ -46,6 +47,16 @@ FolderWatcher::FolderWatcher(const QString &root, QObject *parent) FolderWatcher::~FolderWatcher() { } +void FolderWatcher::setIgnoreHidden(bool ignore) +{ + _ignoreHidden = ignore; +} + +bool FolderWatcher::ignoreHidden() +{ + return _ignoreHidden; +} + void FolderWatcher::addIgnoreListFile( const QString& file ) { if( file.isEmpty() ) return; @@ -71,10 +82,14 @@ bool FolderWatcher::pathIsIgnored( const QString& path ) { if( path.isEmpty() ) return true; - QFileInfo fInfo(path); - if( fInfo.isHidden() ) { - qDebug() << "* Discarded as is hidden!" << fInfo.filePath(); - return true; + // if events caused by changes to hidden files should be ignored, a QFileInfo + // object will tell us if the file is hidden + if( _ignoreHidden ) { + QFileInfo fInfo(path); + if( fInfo.isHidden() ) { + qDebug() << "* Discarded as is hidden!" << fInfo.filePath(); + return true; + } } // TODO: Best use csync_excluded_no_ctx() here somehow! diff --git a/src/gui/folderwatcher.h b/src/gui/folderwatcher.h index 7a7a41dbc2..b0f529940e 100644 --- a/src/gui/folderwatcher.h +++ b/src/gui/folderwatcher.h @@ -77,6 +77,10 @@ public: /* Check if the path is ignored. */ bool pathIsIgnored( const QString& path ); + /* set if the folderwatcher ignores events of hidden files */ + void setIgnoreHidden(bool ignore); + bool ignoreHidden(); + signals: /** Emitted when one of the watched directories or one * of the contained files is changed. */ @@ -98,6 +102,7 @@ private: QStringList _ignores; QTime _timer; QSet _lastPaths; + bool _ignoreHidden; friend class FolderWatcherPrivate; }; From 3ba6e20f6623b2d2c450e4214fe13bfd9445663b Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Tue, 14 Jul 2015 16:56:55 +0200 Subject: [PATCH 12/19] Folder: Getter and Setter for ignore hidden files for folder config. With these, the ignore hidden files flag can be set and get from the folder configuration file. --- src/gui/folder.cpp | 11 +++++++++++ src/gui/folder.h | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index f83ad78834..6caae6b337 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -191,6 +191,17 @@ QString Folder::path() const return p; } +bool Folder::ignoreHiddenFiles() +{ + bool re(_definition.ignoreHiddenFiles); + return re; +} + +void Folder::setIgnoreHiddenFiles(bool ignore) +{ + _definition.ignoreHiddenFiles = ignore; +} + QString Folder::cleanPath() { QString cleanedPath = QDir::cleanPath(_definition.localPath); diff --git a/src/gui/folder.h b/src/gui/folder.h index 837cae4209..d500b0c4fe 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -170,6 +170,13 @@ public: void setDirtyNetworkLimits(); + /** + * Ignore syncing of hidden files or not. This is defined in the + * folder definition + */ + bool ignoreHiddenFiles(); + void setIgnoreHiddenFiles(bool ignore); + // Used by the Socket API SyncJournalDb *journalDb() { return &_journal; } From 7195e40ffa0d4a6f4e1fb64f447c6375a0d72fe5 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Tue, 14 Jul 2015 16:58:06 +0200 Subject: [PATCH 13/19] FolderMan: Set ignore hidden files in the folder watcher. The flag is read from the folder configuration and passed on to the folder watcher so that it ignores events to hidden files accordingly. --- src/gui/folderman.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index 546d00d3b9..765a877fce 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -151,6 +151,7 @@ void FolderMan::registerFolderMonitor( Folder *folder ) ConfigFile cfg; fw->addIgnoreListFile( cfg.excludeFile(ConfigFile::SystemScope) ); fw->addIgnoreListFile( cfg.excludeFile(ConfigFile::UserScope) ); + fw->setIgnoreHidden( folder->ignoreHiddenFiles() ); // Connect the pathChanged signal, which comes with the changed path, // to the signal mapper which maps to the folder alias. The changed path From 27e0f9d132cd12de643a89a641855ad999cfd66a Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 10:48:00 +0200 Subject: [PATCH 14/19] csync: Updated and added some documentation. --- csync/src/csync.h | 53 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/csync/src/csync.h b/csync/src/csync.h index 99e3695d0a..60a8d5c074 100644 --- a/csync/src/csync.h +++ b/csync/src/csync.h @@ -49,41 +49,37 @@ struct csync_client_certs_s { char *certificatePasswd; }; -/** - * Instruction enum. In the file traversal structure, it describes - * the csync state of a file. - */ enum csync_status_codes_e { CSYNC_STATUS_OK = 0, CSYNC_STATUS_ERROR = 1024, /* don't use this code, */ - CSYNC_STATUS_UNSUCCESSFUL, - CSYNC_STATUS_NO_LOCK, /* OBSOLETE does not happen anymore */ - CSYNC_STATUS_STATEDB_LOAD_ERROR, - CSYNC_STATUS_STATEDB_CORRUPTED, - CSYNC_STATUS_NO_MODULE, - CSYNC_STATUS_TIMESKEW, /* OBSOLETE */ + CSYNC_STATUS_UNSUCCESSFUL, /* Unspecific problem happend */ + CSYNC_STATUS_NO_LOCK, /* OBSOLETE does not happen anymore */ + CSYNC_STATUS_STATEDB_LOAD_ERROR, /* Statedb can not be loaded. */ + CSYNC_STATUS_STATEDB_CORRUPTED, /* Statedb is corrupted */ + CSYNC_STATUS_NO_MODULE, /* URL passed to csync does not start with owncloud:// or ownclouds:// */ + CSYNC_STATUS_TIMESKEW, /* OBSOLETE */ CSYNC_STATUS_FILESYSTEM_UNKNOWN, /* UNUSED */ - CSYNC_STATUS_TREE_ERROR, - CSYNC_STATUS_MEMORY_ERROR, - CSYNC_STATUS_PARAM_ERROR, - CSYNC_STATUS_UPDATE_ERROR, - CSYNC_STATUS_RECONCILE_ERROR, - CSYNC_STATUS_PROPAGATE_ERROR, /* OBSOLETE */ + CSYNC_STATUS_TREE_ERROR, /* csync trees could not be created */ + CSYNC_STATUS_MEMORY_ERROR, /* not enough memory problem */ + CSYNC_STATUS_PARAM_ERROR, /* parameter is zero where not expected */ + CSYNC_STATUS_UPDATE_ERROR, /* general update or discovery error */ + CSYNC_STATUS_RECONCILE_ERROR, /* general reconcile error */ + CSYNC_STATUS_PROPAGATE_ERROR, /* OBSOLETE */ CSYNC_STATUS_REMOTE_ACCESS_ERROR, /* UNUSED */ CSYNC_STATUS_REMOTE_CREATE_ERROR, /* UNUSED */ - CSYNC_STATUS_REMOTE_STAT_ERROR, /* UNUSED */ + CSYNC_STATUS_REMOTE_STAT_ERROR, /* UNUSED */ CSYNC_STATUS_LOCAL_CREATE_ERROR, /* UNUSED */ - CSYNC_STATUS_LOCAL_STAT_ERROR, /* UNUSED */ - CSYNC_STATUS_PROXY_ERROR, /* UNUSED */ - CSYNC_STATUS_LOOKUP_ERROR, - CSYNC_STATUS_SERVER_AUTH_ERROR, - CSYNC_STATUS_PROXY_AUTH_ERROR, - CSYNC_STATUS_CONNECT_ERROR, - CSYNC_STATUS_TIMEOUT, - CSYNC_STATUS_HTTP_ERROR, - CSYNC_STATUS_PERMISSION_DENIED, + CSYNC_STATUS_LOCAL_STAT_ERROR, /* UNUSED */ + CSYNC_STATUS_PROXY_ERROR, /* UNUSED */ + CSYNC_STATUS_LOOKUP_ERROR, /* Neon fails to find proxy. Almost OBSOLETE */ + CSYNC_STATUS_SERVER_AUTH_ERROR, /* UNUSED */ + CSYNC_STATUS_PROXY_AUTH_ERROR, /* UNUSED */ + CSYNC_STATUS_CONNECT_ERROR, /* neon driven connection failed */ + CSYNC_STATUS_TIMEOUT, /* UNUSED */ + CSYNC_STATUS_HTTP_ERROR, /* UNUSED */ + CSYNC_STATUS_PERMISSION_DENIED, /* */ CSYNC_STATUS_NOT_FOUND, CSYNC_STATUS_FILE_EXISTS, CSYNC_STATUS_OUT_OF_SPACE, @@ -120,7 +116,10 @@ typedef enum csync_status_codes_e CSYNC_STATUS; #define CSYNC_STATUS_IS_ERR(x) (unlikely((x) >= CSYNC_STATUS_ERROR)) #define CSYNC_STATUS_IS_EQUAL(x, y) ((x) == (y)) - +/** + * Instruction enum. In the file traversal structure, it describes + * the csync state of a file. + */ enum csync_instructions_e { CSYNC_INSTRUCTION_NONE = 0x00000000, /* Nothing to do (UPDATE|RECONCILE) */ CSYNC_INSTRUCTION_EVAL = 0x00000001, /* There was changed compared to the DB (UPDATE) */ From ddf7419aca05cc1a3f26c0f1a8504ebbe4e111a8 Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 11:29:33 +0200 Subject: [PATCH 15/19] csync: Initialize the hidden files flag properly. --- csync/src/csync.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csync/src/csync.c b/csync/src/csync.c index 0122b5f29b..908612d471 100644 --- a/csync/src/csync.c +++ b/csync/src/csync.c @@ -124,6 +124,8 @@ int csync_create(CSYNC **csync, const char *local, const char *remote) { ctx->abort = false; + ctx->ignore_hidden_files = true; + *csync = ctx; return 0; } From 1aa17cdb69fb27b912f87688d42b46419e3789dd Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 11:30:50 +0200 Subject: [PATCH 16/19] owncloudcmd: add option to not ignore hidden files. --- src/cmd/cmd.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp index d24418d832..a75be0451d 100644 --- a/src/cmd/cmd.cpp +++ b/src/cmd/cmd.cpp @@ -55,6 +55,7 @@ struct CmdOptions { bool trustSSL; bool useNetrc; bool interactive; + bool ignoreHiddenFiles; QString exclude; QString unsyncedfolders; }; @@ -154,6 +155,7 @@ void help() std::cout << " --password, -p [pass] Use [pass] as password" << std::endl; std::cout << " -n Use netrc (5) for login" << std::endl; std::cout << " --non-interactive Do not block execution with interaction" << std::endl; + std::cout << " -h Sync hidden files,do not ignore them" << std::endl; std::cout << " --version, -v Display version and exit" << std::endl; std::cout << "" << std::endl; exit(1); @@ -216,6 +218,8 @@ void parseOptions( const QStringList& app_args, CmdOptions *options ) options->trustSSL = true; } else if( option == "-n") { options->useNetrc = true; + } else if( option == "-h") { + options->ignoreHiddenFiles = false; } else if( option == "--non-interactive") { options->interactive = false; } else if( (option == "-u" || option == "--user") && !it.peekNext().startsWith("-") ) { @@ -269,6 +273,7 @@ int main(int argc, char **argv) { options.trustSSL = false; options.useNetrc = false; options.interactive = true; + options.ignoreHiddenFiles = true; ClientProxy clientProxy; parseOptions( app.arguments(), &options ); @@ -364,6 +369,9 @@ restart_sync: return EXIT_FAILURE; } + // ignore hidden files or not + _csync_ctx->ignore_hidden_files = options.ignoreHiddenFiles; + csync_set_module_property(_csync_ctx, "csync_context", _csync_ctx); if( !options.proxy.isNull() ) { QString host; From cfe5150037d4a895213aa67934d2e8c71a70292b Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 14:50:41 +0200 Subject: [PATCH 17/19] IgnoreListEditor: handle the ignore hidden files checkbox. Write the setting of the checkbox to all folder definitions. --- src/gui/ignorelisteditor.cpp | 23 +++++ src/gui/ignorelisteditor.h | 2 + src/gui/ignorelisteditor.ui | 183 ++++++++++++++++++++--------------- 3 files changed, 129 insertions(+), 79 deletions(-) diff --git a/src/gui/ignorelisteditor.cpp b/src/gui/ignorelisteditor.cpp index 23f96b1a32..77a3bcb08f 100644 --- a/src/gui/ignorelisteditor.cpp +++ b/src/gui/ignorelisteditor.cpp @@ -57,6 +57,13 @@ IgnoreListEditor::IgnoreListEditor(QWidget *parent) : ui->tableWidget->horizontalHeader()->setResizeMode(patternCol, QHeaderView::Stretch); ui->tableWidget->verticalHeader()->setVisible(false); + + /* value for syncing hidden files */ + bool ignoreHidden = true; + if( FolderMan::instance()->map().count() > 0 ) { + ignoreHidden = FolderMan::instance()->map().first()->ignoreHiddenFiles(); + } + ui->ignoreHiddenFilesCheckBox->setChecked( !ignoreHidden ); } IgnoreListEditor::~IgnoreListEditor() @@ -64,6 +71,11 @@ IgnoreListEditor::~IgnoreListEditor() delete ui; } +bool IgnoreListEditor::ignoreHiddenFiles() +{ + return ! ui->ignoreHiddenFilesCheckBox->isChecked(); +} + void IgnoreListEditor::slotItemSelectionChanged() { QTableWidgetItem *item = ui->tableWidget->currentItem(); @@ -110,6 +122,17 @@ void IgnoreListEditor::slotUpdateLocalIgnoreList() QMessageBox::warning(this, tr("Could not open file"), tr("Cannot write changes to '%1'.").arg(ignoreFile)); } + + /* handle the hidden file checkbox */ + bool ignoreHiddenFiles = ! ui->ignoreHiddenFilesCheckBox->isChecked(); + + /* the ignoreHiddenFiles flag is a folder specific setting, but for now, it is + * handled globally. Save it to every folder that is defined. + */ + foreach (Folder* folder, FolderMan::instance()->map()) { + folder->setIgnoreHiddenFiles(ignoreHiddenFiles); + folder->saveToSettings(); + } } void IgnoreListEditor::slotAddPattern() diff --git a/src/gui/ignorelisteditor.h b/src/gui/ignorelisteditor.h index 57bb04a32a..bd64d8a582 100644 --- a/src/gui/ignorelisteditor.h +++ b/src/gui/ignorelisteditor.h @@ -36,6 +36,8 @@ public: explicit IgnoreListEditor(QWidget *parent = 0); ~IgnoreListEditor(); + bool ignoreHiddenFiles(); + private slots: void slotItemSelectionChanged(); void slotRemoveCurrentItem(); diff --git a/src/gui/ignorelisteditor.ui b/src/gui/ignorelisteditor.ui index e6b7c11715..b6d33c74f0 100644 --- a/src/gui/ignorelisteditor.ui +++ b/src/gui/ignorelisteditor.ui @@ -6,96 +6,121 @@ 0 0 - 471 - 359 + 438 + 463 Ignored Files Editor - - + + + + + Global Ignore Settings + + + + + + Sync hidden files + + + + + + + + + + Files Ingored by Patterns + + + + + + true + + + Remove + + + + + + + true + + + Add + + + + + + + true + + + QAbstractItemView::SingleSelection + + + QAbstractItemView::SelectRows + + + 2 + + + + Pattern + + + + + Allow Deletion + + + + + + + + true + + + Qt::Vertical + + + + 20 + 213 + + + + + + + + true + + + + + + true + + + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - - - true - - - Qt::Vertical - - - - 20 - 213 - - - - - - - - true - - - Add - - - - - - - true - - - Remove - - - - - - - true - - - - - - true - - - - - - - true - - - QAbstractItemView::SingleSelection - - - QAbstractItemView::SelectRows - - - 2 - - - - Pattern - - - - - Allow Deletion - - - - From a5528b7cd4e273b105da2984ec2631422a35ea2b Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 14:51:37 +0200 Subject: [PATCH 18/19] AccountSettings: When adding a new folder, set ignore hidden flag. The value is taken from existing folder definitions, or defaults to true. --- src/gui/accountsettings.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 6bf8fd45bb..f00d847d49 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -192,6 +192,17 @@ void AccountSettings::slotFolderWizardAccepted() definition.alias = folderWizard->field(QLatin1String("alias")).toString(); definition.localPath = folderWizard->field(QLatin1String("sourceFolder")).toString(); definition.targetPath = folderWizard->property("targetPath").toString(); + + bool ignoreHidden = true; + /* take the value from the definition of already existing folders. All folders have + * the same setting so far, that's why it's ok to check the first one. + * The default is to not sync hidden files + */ + if( folderMan->map().count() > 0) { + ignoreHidden = folderMan->map().first()->ignoreHiddenFiles(); + } + definition.ignoreHiddenFiles = ignoreHidden; + auto selectiveSyncBlackList = folderWizard->property("selectiveSyncBlackList").toStringList(); folderMan->setSyncEnabled(true); From c72b6fb8e6c73d90f4be0bd53089e42db562044d Mon Sep 17 00:00:00 2001 From: Klaas Freitag Date: Wed, 15 Jul 2015 15:01:27 +0200 Subject: [PATCH 19/19] csync_update: Reorg the code a bit to make it easier to read and understand --- csync/src/csync_update.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index 3fd98eb412..cf73f3185a 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -166,7 +166,17 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, /* Check if file is excluded */ excluded = csync_excluded(ctx, path,type); - if (excluded != CSYNC_NOT_EXCLUDED) { + if( excluded == CSYNC_NOT_EXCLUDED ) { + /* Even if it is not excluded by a pattern, maybe it is to be ignored + * because it's a hidden file that should not be synced. + * This code should probably be in csync_exclude, but it does not have the fs parameter. + * Keep it here for now */ + if (ctx->ignore_hidden_files && (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN)) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); + excluded = CSYNC_FILE_EXCLUDE_HIDDEN; + } + } else { + /* File is ignored because it's matched by a user- or system exclude pattern. */ CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded); if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE) { return 1; @@ -174,13 +184,6 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, if (excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { return 1; } - } else { - /* This code should probably be in csync_exclude, but it does not have the fs parameter. - Keep it here for now */ - if (ctx->ignore_hidden_files && (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN)) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); - excluded = CSYNC_FILE_EXCLUDE_HIDDEN; - } } if (ctx->current == REMOTE_REPLICA && ctx->callbacks.checkSelectiveSyncBlackListHook) {