From af7e36422eb52ecd768329c6420caef6d36da489 Mon Sep 17 00:00:00 2001 From: Olivier Goffart Date: Fri, 26 Jul 2013 12:37:54 +0200 Subject: [PATCH] Add ignore files that need cleanup Add the possibility to ignore files but specify they should be deleted if the directory is deleted. --- config/ocsync_exclude.conf | 2 +- src/csync.c | 92 ++++++++++++++++----------------- src/csync_exclude.c | 20 +++++-- src/csync_exclude.h | 2 +- src/csync_private.h | 2 + src/csync_propagate.c | 28 ++++++++++ src/csync_update.c | 18 ++++++- tests/ownCloud/ownCloud/Test.pm | 13 ++--- tests/ownCloud/t4.pl | 25 +++++++-- 9 files changed, 138 insertions(+), 64 deletions(-) mode change 100644 => 100755 tests/ownCloud/t4.pl diff --git a/config/ocsync_exclude.conf b/config/ocsync_exclude.conf index 9cc35edb58..f3f8b36afe 100644 --- a/config/ocsync_exclude.conf +++ b/config/ocsync_exclude.conf @@ -19,7 +19,7 @@ .thumbnails *.~* -.directory +]*.directory # exclude all object file # *.o diff --git a/src/csync.c b/src/csync.c index 88480bcd68..4dd61d04a8 100644 --- a/src/csync.c +++ b/src/csync.c @@ -750,6 +750,50 @@ static int _merge_and_write_statedb(CSYNC *ctx) { return rc; } +/* reset all the list to empty. + * used by csync_commit and csync_destroy */ +static void _csync_clean_ctx(CSYNC *ctx) +{ + c_list_t * walk; + + while (ctx->progress_info) { + csync_progressinfo_t *next = ctx->progress_info->next; + csync_statedb_free_progressinfo(ctx->progress_info); + ctx->progress_info = next; + } + + /* destroy the rbtrees */ + if (c_rbtree_size(ctx->local.tree) > 0) { + c_rbtree_destroy(ctx->local.tree, _tree_destructor); + } + + if (c_rbtree_size(ctx->remote.tree) > 0) { + c_rbtree_destroy(ctx->remote.tree, _tree_destructor); + } + + csync_rename_destroy(ctx); + + for (walk = c_list_last(ctx->local.ignored_cleanup); walk != NULL; walk = c_list_prev(walk)) { + SAFE_FREE(walk->data); + } + for (walk = c_list_last(ctx->remote.ignored_cleanup); walk != NULL; walk = c_list_prev(walk)) { + SAFE_FREE(walk->data); + } + + /* free memory */ + c_rbtree_free(ctx->local.tree); + c_list_free(ctx->local.list); + c_list_free(ctx->local.ignored_cleanup); + c_rbtree_free(ctx->remote.tree); + c_list_free(ctx->remote.list); + c_list_free(ctx->remote.ignored_cleanup); + + ctx->remote.list = 0; + ctx->local.list = 0; + ctx->remote.ignored_cleanup = 0; + ctx->local.ignored_cleanup = 0; +} + int csync_commit(CSYNC *ctx) { int rc = 0; char *lock = NULL; @@ -772,31 +816,7 @@ int csync_commit(CSYNC *ctx) { csync_vio_commit(ctx); - while (ctx->progress_info) { - csync_progressinfo_t *next = ctx->progress_info->next; - csync_statedb_free_progressinfo(ctx->progress_info); - ctx->progress_info = next; - } - - /* destroy the rbtrees */ - if (c_rbtree_size(ctx->local.tree) > 0) { - c_rbtree_destroy(ctx->local.tree, _tree_destructor); - } - - if (c_rbtree_size(ctx->remote.tree) > 0) { - c_rbtree_destroy(ctx->remote.tree, _tree_destructor); - } - - csync_rename_destroy(ctx); - - /* free memory */ - c_rbtree_free(ctx->local.tree); - c_list_free(ctx->local.list); - c_rbtree_free(ctx->remote.tree); - c_list_free(ctx->remote.list); - - ctx->remote.list = 0; - ctx->local.list = 0; + _csync_clean_ctx(ctx); ctx->remote.read_from_db = 0; @@ -883,28 +903,8 @@ int csync_destroy(CSYNC *ctx) { csync_lock_remove(ctx, lock); } - while (ctx->progress_info) { - csync_progressinfo_t *next = ctx->progress_info->next; - csync_statedb_free_progressinfo(ctx->progress_info); - ctx->progress_info = next; - } + _csync_clean_ctx(ctx); - /* destroy the rbtrees */ - if (c_rbtree_size(ctx->local.tree) > 0) { - c_rbtree_destroy(ctx->local.tree, _tree_destructor); - } - - if (c_rbtree_size(ctx->remote.tree) > 0) { - c_rbtree_destroy(ctx->remote.tree, _tree_destructor); - } - - csync_rename_destroy(ctx); - - /* free memory */ - c_rbtree_free(ctx->local.tree); - c_list_free(ctx->local.list); - c_rbtree_free(ctx->remote.tree); - c_list_free(ctx->remote.list); SAFE_FREE(ctx->local.uri); SAFE_FREE(ctx->remote.uri); SAFE_FREE(ctx->options.config_dir); diff --git a/src/csync_exclude.c b/src/csync_exclude.c index 8e403a67a2..63950645cc 100644 --- a/src/csync_exclude.c +++ b/src/csync_exclude.c @@ -135,6 +135,8 @@ int csync_excluded(CSYNC *ctx, const char *path) { char *bname; int rc; int match = 0; + const char *it; + int type = 0; /* exclude the lock file */ if (c_streq( path, CSYNC_LOCK_FILE )) { @@ -180,14 +182,22 @@ int csync_excluded(CSYNC *ctx, const char *path) { } for (i = 0; match == 0 && i < ctx->excludes->count; i++) { - rc = csync_fnmatch(ctx->excludes->vector[i], path, 0); - if (rc == 0) { - match = 1; + it = ctx->excludes->vector[i]; + type = 1; + /* Ecludes starting with ']' means it can be cleanup */ + if (it[0] == ']') { + ++it; + type = 2; } - rc = csync_fnmatch(ctx->excludes->vector[i], bname, 0); + rc = csync_fnmatch(it, path, 0); if (rc == 0) { - match = 1; + match = type; + } + + rc = csync_fnmatch(it, bname, 0); + if (rc == 0) { + match = type; } } diff --git a/src/csync_exclude.h b/src/csync_exclude.h index 6ca3476e4d..419cfdeb95 100644 --- a/src/csync_exclude.h +++ b/src/csync_exclude.h @@ -53,7 +53,7 @@ void csync_exclude_destroy(CSYNC *ctx); * @param ctx The synchronizer context. * @param path The patch to check. * - * @return 1 if excluded, 0 if not. + * @return 2 if excluded and needs cleanup, 1 if excluded, 0 if not. */ int csync_excluded(CSYNC *ctx, const char *path); diff --git a/src/csync_private.h b/src/csync_private.h index 225b63a3c3..34857519b5 100644 --- a/src/csync_private.h +++ b/src/csync_private.h @@ -106,6 +106,7 @@ struct csync_s { c_rbtree_t *tree; c_list_t *list; enum csync_replica_e type; + c_list_t *ignored_cleanup; } local; struct { @@ -114,6 +115,7 @@ struct csync_s { c_list_t *list; enum csync_replica_e type; int read_from_db; + c_list_t *ignored_cleanup; } remote; struct { diff --git a/src/csync_propagate.c b/src/csync_propagate.c index 62f006e141..2c647bb9c1 100644 --- a/src/csync_propagate.c +++ b/src/csync_propagate.c @@ -1441,16 +1441,20 @@ static int _cmp_char( const void *d1, const void *d2 ) static int _csync_propagation_cleanup(CSYNC *ctx) { c_list_t *list = NULL; c_list_t *walk = NULL; + c_list_t *walk2 = NULL; + c_list_t *ignored_cleanup = NULL; char *uri = NULL; char *dir = NULL; switch (ctx->current) { case LOCAL_REPLICA: list = ctx->local.list; + ignored_cleanup = ctx->local.ignored_cleanup; uri = ctx->local.uri; break; case REMOTE_REPLICA: list = ctx->remote.list; + ignored_cleanup = ctx->remote.ignored_cleanup; uri = ctx->remote.uri; break; default: @@ -1473,10 +1477,34 @@ static int _csync_propagation_cleanup(CSYNC *ctx) { pst = (csync_file_stat_t **) walk->data; st = *(pst); + + /* Cleanup ignored files */ + for (walk2 = c_list_last(ignored_cleanup); walk2 != NULL; walk2 = c_list_prev(walk2)) { + const char *fn = (const char*) walk2->data; + /* check if the file name does not starts with the path to remove. */ + if (strlen(fn) < st->pathlen || fn[st->pathlen] != '/' + || strncmp(fn, st->path, st->pathlen) != 0) { + continue; + } + + if (asprintf(&dir, "%s/%s", uri, fn) < 0) { + return -1; + } + + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Removing ignored file %s ", dir); + + if (csync_vio_unlink(ctx, dir) < 0) { + return -1; + } + + SAFE_FREE(dir); + } + if (asprintf(&dir, "%s/%s", uri, st->path) < 0) { return -1; } + if (csync_vio_rmdir(ctx, dir) < 0) { _csync_remove_error(ctx, st, uri); } else { diff --git a/src/csync_update.c b/src/csync_update.c index 196b7c6a1e..a34a7c5e38 100644 --- a/src/csync_update.c +++ b/src/csync_update.c @@ -435,6 +435,7 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, while ((dirent = csync_vio_readdir(ctx, dh))) { const char *path = NULL; int flag; + int excluded; d_name = dirent->name; if (d_name == NULL) { @@ -470,8 +471,21 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, } /* Check if file is excluded */ - if (csync_excluded(ctx, path)) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded", path); + excluded = csync_excluded(ctx, path); + if (excluded) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", path, excluded); + if (excluded == 2) { + switch (ctx->current) { + case LOCAL_REPLICA: + ctx->local.ignored_cleanup = c_list_append(ctx->local.ignored_cleanup, c_strdup(path)); + break; + case REMOTE_REPLICA: + ctx->remote.ignored_cleanup = c_list_append(ctx->remote.ignored_cleanup, c_strdup(path)); + break; + default: + break; + } + } csync_vio_file_stat_destroy(dirent); dirent = NULL; SAFE_FREE(filename); diff --git a/tests/ownCloud/ownCloud/Test.pm b/tests/ownCloud/ownCloud/Test.pm index 2f67385866..644068a4e5 100644 --- a/tests/ownCloud/ownCloud/Test.pm +++ b/tests/ownCloud/ownCloud/Test.pm @@ -49,7 +49,7 @@ our $localDir = "turbo"; @ISA = qw(Exporter); @EXPORT = qw( initTesting createRemoteDir createLocalDir cleanup csync assertLocalDirs assertLocalAndRemoteDir - glob_put put_to_dir localDir remoteDir localCleanup createLocalFile); + glob_put put_to_dir localDir remoteDir localCleanup createLocalFile remoteCleanup); sub fromFileName($) { @@ -137,21 +137,22 @@ sub cleanup() print "\nInterrupt before cleanup in 4 seconds...\n"; sleep(4); - remoteCleanup( ); + remoteCleanup( '' ); localCleanup( '' ); } -sub remoteCleanup( ) +sub remoteCleanup($) { - $d->open( -url => $owncloud ); + my ($dir) = @_; + $d->open( -url => $owncloud . $dir ); print "Cleaning Remote!\n"; - my $re = $d->delete( $owncloud ); + my $re = $d->delete( $owncloud . $dir ); if( $re == 0 ) { - print "Failed to clenup directory <$owncloud>\n"; + print "Failed to clenup directory <$owncloud $dir>\n"; } return $re; } diff --git a/tests/ownCloud/t4.pl b/tests/ownCloud/t4.pl old mode 100644 new mode 100755 index 7f0287a463..959fd93ac3 --- a/tests/ownCloud/t4.pl +++ b/tests/ownCloud/t4.pl @@ -15,7 +15,7 @@ use ownCloud::Test; use strict; -print "Hello, this is t4, a tester for files that cannot be stated\n"; +print "Hello, this is t4, a tester for A) files that cannot be stated and B) excluded files\n"; # stat error occours on windsows when the file is busy for example initTesting(); @@ -30,7 +30,7 @@ csync(); # Check if the files from toremote1 are now in t1/remoteToLocal1 # they should have taken the way via the ownCloud. print "Assert the local file copy\n"; -assertLocalAndRemoteDir( 'test_stat', 0 ); +assertLocalAndRemoteDir( '', 0 ); system( "echo foobar2 >> " . localDir() . 'test_stat/file.txt' ); @@ -42,6 +42,9 @@ csync(); # TODO: some check here. + +print("Restore the original rights"); + system( "chmod 700 " . localDir() . 'test_stat' ); system( "echo foobar3 >> " . localDir() . 'test_stat/file.txt' ); @@ -49,10 +52,26 @@ csync(); print "Check if everything is still the same\n"; -assertLocalAndRemoteDir( 'test_stat', 0 ); +assertLocalAndRemoteDir( '', 0 ); # TODO: Check that the file content is fine on the server and that there was no conflict +print("Added a file that is on the ignore list\n"); +# (*.directory is in the ignored list that needs cleanup) +# (it is names with _conflict) because i want the conflicft detection of assertLocalAndRemoteDir to work +system( "echo dir >> " . localDir() . 'test_stat/file_conflict.directory' ); +csync(); +# The file_conflict.directory is seen as a conflict +assertLocalAndRemoteDir( '', 1 ); +# TODO: check that the file_conflict.directory is indeed NOT on the server + +print("Remove a directory containing a local file\n"); +remoteCleanup('test_stat'); +csync(); +assertLocalAndRemoteDir( '', 0 ); + + + cleanup(); # --