diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d666b0ff0..dac9cdf88e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,11 +8,11 @@ set(APPLICATION_NAME ${PROJECT_NAME}) set(APPLICATION_VERSION_MAJOR "0") set(APPLICATION_VERSION_MINOR "50") -set(APPLICATION_VERSION_PATCH "8") +set(APPLICATION_VERSION_PATCH "10") set(APPLICATION_VERSION "${APPLICATION_VERSION_MAJOR}.${APPLICATION_VERSION_MINOR}.${APPLICATION_VERSION_PATCH}") -set(LIBRARY_VERSION "0.1.8") +set(LIBRARY_VERSION "0.1.10") set(LIBRARY_SOVERSION "0") # where to look first for cmake modules, before ${CMAKE_ROOT}/Modules/ is checked diff --git a/CPackConfig.cmake b/CPackConfig.cmake index a95c1e1016..131f105f02 100644 --- a/CPackConfig.cmake +++ b/CPackConfig.cmake @@ -12,7 +12,7 @@ set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/COPYING") ### versions set(CPACK_PACKAGE_VERSION_MAJOR "0") -set(CPACK_PACKAGE_VERSION_MINOR "49") +set(CPACK_PACKAGE_VERSION_MINOR "50") set(CPACK_PACKAGE_VERSION_PATCH "9") set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") diff --git a/modules/csync_owncloud.c b/modules/csync_owncloud.c index 4f5cc22f08..4163e32efb 100644 --- a/modules/csync_owncloud.c +++ b/modules/csync_owncloud.c @@ -82,6 +82,7 @@ typedef unsigned long dav_size_t; /* Struct to store data for each resource found during an opendir operation. * It represents a single file entry. */ + typedef struct resource { char *uri; /* The complete uri */ char *name; /* The filename only */ @@ -89,6 +90,7 @@ typedef struct resource { enum resource_type type; dav_size_t size; time_t modtime; + char* md5; struct resource *next; } resource; @@ -117,6 +119,8 @@ struct transfer_context { const char *method; /* the HTTP method, either PUT or GET */ ne_decompress *decompress; /* the decompress context */ int fileWritten; /* flag to indicate that a buffer file was written for PUTs */ + char *md5; + char *clean_uri; }; /* Struct with the WebDAV session */ @@ -131,10 +135,12 @@ struct dav_session_s { char *proxy_user; char *proxy_pwd; - time_t prev_delta; - time_t time_delta; /* The time delta to use. */ + char *session_key; + + long int prev_delta; + long int time_delta; /* The time delta to use. */ long int time_delta_sum; /* What is the time delta average? */ - int time_delta_cnt; /* How often was the server time gathered? */ + long int time_delta_cnt; /* How often was the server time gathered? */ }; /* The list of properties that is fetched in PropFind on a collection */ @@ -142,6 +148,7 @@ static const ne_propname ls_props[] = { { "DAV:", "getlastmodified" }, { "DAV:", "getcontentlength" }, { "DAV:", "resourcetype" }, + { "DAV:", "getetag"}, { NULL, NULL } }; @@ -405,6 +412,120 @@ static int configureProxy( ne_session *session ) return re; } +/* + * This hook is called for with the response of a request. Here its checked + * if a Set-Cookie header is there for the PHPSESSID. The key is stored into + * the webdav session to be added to subsequent requests. + */ +#define PHPSESSID "PHPSESSID=" +static void post_request_hook(ne_request *req, void *userdata, const ne_status *status) +{ + const char *set_cookie_header = NULL; + const char *sc = NULL; + char *key = NULL; + + (void) userdata; + + if(!(status && req)) return; + if( status->klass == 2 || status->code == 401 ) { + /* successful request */ + set_cookie_header = ne_get_response_header( req, "Set-Cookie" ); + if( set_cookie_header ) { + DEBUG_WEBDAV(" Set-Cookie found: %s", set_cookie_header); + /* try to find a ', ' sequence which is the separator of neon if multiple Set-Cookie + * headers are there. + * The following code parses a string like this: + * PHPSESSID=n8feu3dsbarpvvufqae9btn5fl7m7ikgh5ml1fg37v4i2cah7k41; path=/; HttpOnly */ + sc = set_cookie_header; + while(sc) { + if( strlen(sc) > strlen(PHPSESSID) && + strncmp( sc, PHPSESSID, strlen(PHPSESSID)) == 0 ) { + const char *sc_val = sc; /* + strlen(PHPSESSID); */ + const char *sc_end = sc_val; + int cnt = 0; + int len = strlen(sc_val); /* The length of the rest of the header string. */ + + while( cnt < len && *sc_end != ';' && *sc_end != ',') { + cnt++; + sc_end++; + } + if( cnt == len ) { + /* exit: We are at the end. */ + sc = NULL; + } else if( *sc_end == ';' ) { + /* We are at the end of the session key. */ + int keylen = sc_end-sc_val; + if( key ) SAFE_FREE(key); + key = c_malloc(keylen+1); + strncpy( key, sc_val, keylen ); + key[keylen] = '\0'; + + /* now search for a ',' to find a potential other header entry */ + while(cnt < len && *sc_end != ',') { + cnt++; + sc_end++; + } + if( cnt < len ) + sc = sc_end+2; /* mind the space after the comma */ + else + sc = NULL; + } else if( *sc_end == ',' ) { + /* A new entry is to check. */ + if( *(sc_end + 1) == ' ') { + sc = sc_end+2; + } else { + /* error condition */ + sc = NULL; + } + } + } else { + /* It is not a PHPSESSID-Header but another one which we're not interested in. + * forward to next header entry (search ',' ) + */ + int len = strlen(sc); + int cnt = 0; + + while(cnt < len && *sc != ',') { + cnt++; + sc++; + } + if( cnt < len ) + sc = sc+2; /* mind the space after the comma */ + else + sc = NULL; + } + } + } + } else { + DEBUG_WEBDAV("Request failed, don't take session header."); + } + if( key ) { + DEBUG_WEBDAV("----> Session-key: %s", key); + SAFE_FREE(dav_session.session_key); + dav_session.session_key = key; + } +} + +/* + * this hook is called just after a request has been created, before its sent. + * Here it is used to set the session cookie if available. + */ +static void request_created_hook(ne_request *req, void *userdata, + const char *method, const char *requri) +{ + (void) userdata; + (void) method; + (void) requri; + + if( !req ) return; + + if(dav_session.session_key) { + /* DEBUG_WEBDAV("Setting PHPSESSID to %s", dav_session.session_key); */ + ne_add_request_header(req, "Cookie", dav_session.session_key); + } + +} + /* * Connect to a DAV server * This function sets the flag _connected if the connection is established @@ -490,6 +611,13 @@ static int dav_connect(const char *base_url) { } ne_redirect_register( dav_session.ctx ); + /* Hook to get the Session ID */ + ne_hook_post_headers( dav_session.ctx, post_request_hook, NULL ); + /* Hook called when a request is built. It sets the PHPSESSID header */ + ne_hook_create_request( dav_session.ctx, request_created_hook, NULL ); + + dav_session.session_key = NULL; + /* Proxy support */ proxystate = configureProxy( dav_session.ctx ); if( proxystate < 0 ) { @@ -522,6 +650,7 @@ static void results(void *userdata, struct resource *newres = 0; const char *clength, *modtime = NULL; const char *resourcetype = NULL; + const char *md5sum = NULL; const ne_status *status = NULL; char *path = ne_path_unescape( uri->path ); @@ -561,6 +690,7 @@ static void results(void *userdata, modtime = ne_propset_value( set, &ls_props[0] ); clength = ne_propset_value( set, &ls_props[1] ); resourcetype = ne_propset_value( set, &ls_props[2] ); + md5sum = ne_propset_value( set, &ls_props[3] ); newres->type = resr_normal; if( clength == NULL && resourcetype && strncmp( resourcetype, "", 16 ) == 0) { @@ -579,6 +709,16 @@ static void results(void *userdata, } } + if( md5sum ) { + int len = strlen(md5sum)-2; + if( len > 0 ) { + /* Skip the " around the string coming back from the ne_propset_value call */ + newres->md5 = c_malloc(len+1); + strncpy( newres->md5, md5sum+1, len ); + newres->md5[len] = '\0'; + } + } + /* prepend the new resource to the result list */ newres->next = fetchCtx->list; fetchCtx->list = newres; @@ -686,11 +826,14 @@ static csync_vio_file_stat_t *resourceToFileStat( struct resource *res ) } /* Correct the mtime of the file with the server time delta */ - lfs->mtime = res->modtime; + lfs->mtime = res->modtime - dav_session.time_delta; lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME; lfs->size = res->size; lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE; - + if( res->md5 ) { + lfs->md5 = c_strdup(res->md5); + } + lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; return lfs; } @@ -745,6 +888,7 @@ static void free_fetchCtx( struct listdir_context *ctx ) while( res ) { SAFE_FREE(res->uri); SAFE_FREE(res->name); + SAFE_FREE(res->md5); newres = res->next; SAFE_FREE(res); @@ -753,6 +897,22 @@ static void free_fetchCtx( struct listdir_context *ctx ) SAFE_FREE(ctx); } +static void fill_stat_cache( csync_vio_file_stat_t *lfs ) { + + if( _fs.name ) SAFE_FREE(_fs.name); + if( _fs.md5 ) SAFE_FREE(_fs.md5 ); + + if( !lfs) return; + + _fs.name = c_strdup(lfs->name); + _fs.mtime = lfs->mtime; + _fs.fields = lfs->fields; + _fs.type = lfs->type; + _fs.size = lfs->size; + if( lfs->md5 ) { + _fs.md5 = c_strdup(lfs->md5); + } +} /* * file functions @@ -786,6 +946,7 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) { * stat. If the cache matches, a http call is saved. */ if( _fs.name && strcmp( buf->name, _fs.name ) == 0 ) { + buf->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE; @@ -795,10 +956,16 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) { buf->fields = _fs.fields; buf->type = _fs.type; buf->mtime = _fs.mtime; - buf->size = _fs.size; buf->mode = _stat_perms( _fs.type ); + buf->md5 = NULL; + if( _fs.md5 ) { + buf->md5 = c_strdup( _fs.md5 ); + buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; + } + DEBUG_WEBDAV("stat results from fs cache - md5: %s", _fs.md5 ? _fs.md5 : "NULL"); } else { + DEBUG_WEBDAV("stat results fetched."); /* fetch data via a propfind call. */ fetchCtx = c_malloc( sizeof( struct listdir_context )); if( ! fetchCtx ) { @@ -809,7 +976,6 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) { curi = _cleanPath( uri ); - DEBUG_WEBDAV("I have no stat cache, call propfind for %s", curi ); fetchCtx->list = NULL; fetchCtx->target = curi; fetchCtx->include_target = 1; @@ -859,21 +1025,31 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) { buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_PERMISSIONS; + buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; buf->fields = lfs->fields; buf->type = lfs->type; buf->mtime = lfs->mtime; buf->size = lfs->size; buf->mode = _stat_perms( lfs->type ); + buf->md5 = NULL; + if( lfs->md5 ) { + buf->md5 = c_strdup( lfs->md5 ); + } + /* put the stat information to cache for subsequent calls */ + fill_stat_cache( lfs ); + + /* fill the static stat buf as input for the stat function */ csync_vio_file_stat_destroy( lfs ); } free_fetchCtx( fetchCtx ); } + DEBUG_WEBDAV("STAT result from propfind: %s, md5: %s", buf->name ? buf->name:"NULL", + buf->md5 ? buf->md5 : "NULL" ); } - DEBUG_WEBDAV("STAT result: %s, type=%d", buf->name ? buf->name:"NULL", - buf->type ); + return 0; } @@ -1004,9 +1180,9 @@ static char*_lastDir = NULL; * bool time_sync_required - oC does not require the time sync * int unix_extensions - oC supports unix extensions. */ -static csync_vio_capabilities_t _owncloud_capabilities = { true, false, true, 1 }; +static csync_vio_capabilities_t _owncloud_capabilities = { true, false, false, 1 }; -static csync_vio_capabilities_t *owncloud_get_capabilities(void) +static csync_vio_capabilities_t *owncloud_capabilities(void) { #ifdef _WIN32 _owncloud_capabilities.unix_extensions = 0; @@ -1014,6 +1190,65 @@ static csync_vio_capabilities_t *owncloud_get_capabilities(void) return &_owncloud_capabilities; } +static const char* owncloud_file_id( const char *path ) +{ + ne_request *req = NULL; + const char *header = NULL; + char *uri = _cleanPath(path); + char *buf = NULL; + const char *cbuf = NULL; + csync_vio_file_stat_t *fs = NULL; + bool doHeadRequest= false; /* ownCloud server doesn't have good support for HEAD yet */ + + if( doHeadRequest ) { + /* Perform an HEAD request to the resource. HEAD delivers the + * ETag header back. */ + req = ne_request_create(dav_session.ctx, "HEAD", uri); + ne_request_dispatch(req); + + header = ne_get_response_header(req, "etag"); + } + /* If the request went wrong or the server did not respond correctly + * (that can happen for collections) a stat call is done which translates + * into a PROPFIND request. + */ + if( ! header ) { + /* Clear the cache */ + fill_stat_cache(NULL); + + /* ... and do a stat call. */ + fs = csync_vio_file_stat_new(); + if(fs == NULL) { + DEBUG_WEBDAV( "owncloud_file_id: memory fault."); + errno = ENOMEM; + return NULL; + } + if( owncloud_stat( path, fs ) == 0 ) { + header = fs->md5; + } + } + + /* In case the result is surrounded by "" cut them away. */ + if( header ) { + if( header [0] == '"' && header[ strlen(header)-1] == '"') { + int len = strlen( header )-2; + buf = c_malloc( len+1 ); + strncpy( buf, header+1, len ); + buf[len] = '\0'; + cbuf = buf; + /* do not free header here, as it belongs to the request */ + } else { + cbuf = c_strdup(header); + } + } + DEBUG_WEBDAV("Get file ID for %s: %s", path, cbuf ? cbuf:""); + if( fs ) csync_vio_file_stat_destroy(fs); + if( req ) ne_request_destroy(req); + SAFE_FREE(uri); + + return cbuf; +} + static csync_vio_method_handle_t *owncloud_open(const char *durl, int flags, mode_t mode) { @@ -1031,6 +1266,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl, const char *winTmpUtf8 = NULL; csync_stat_t sb; #endif + const char *etag_header = NULL; struct transfer_context *writeCtx = NULL; csync_vio_file_stat_t statBuf; @@ -1074,6 +1310,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl, } else { if( owncloud_stat( dir, &statBuf ) == 0 ) { SAFE_FREE(statBuf.name); + SAFE_FREE(statBuf.md5); DEBUG_WEBDAV("Directory of file to open exists."); SAFE_FREE( _lastDir ); _lastDir = c_strdup(dir); @@ -1092,6 +1329,8 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl, writeCtx = c_malloc( sizeof(struct transfer_context) ); writeCtx->bytes_written = 0; + writeCtx->clean_uri = c_strdup(uri); + if( rc == NE_OK ) { /* open a temp file to store the incoming data */ #ifdef _WIN32 @@ -1149,7 +1388,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl, writeCtx->fileWritten = 0; /* flag to indicate if contents was pushed to file */ writeCtx->req = ne_request_create(dav_session.ctx, "PUT", uri); - writeCtx->method = "PUT"; + writeCtx->method = "PUT"; } @@ -1197,6 +1436,13 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl, ne_decompress_destroy( writeCtx->decompress ); } + /* preserve the ETag header */ + etag_header = ne_get_response_header( writeCtx->req, "ETag" ); + if( etag_header ) { + writeCtx->md5 = c_strdup( etag_header ); + DEBUG_WEBDAV("GET Etag: %s", etag_header ); + } + /* delete the request in any case */ ne_request_destroy(writeCtx->req); #else @@ -1243,7 +1489,7 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) { int rc; int ret = 0; size_t len = 0; - const _TCHAR *tmpFileName = 0; + const _TCHAR *tmpFileName = NULL; writeCtx = (struct transfer_context*) fhandle; @@ -1302,15 +1548,17 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) { } if (rc == NE_OK) { if ( ne_get_status( writeCtx->req )->klass != 2 ) { - DEBUG_WEBDAV("Error - PUT status value no 2xx"); - errno = EIO; - ret = -1; + // DEBUG_WEBDAV("Error - PUT status value no 2xx"); + // errno = EIO; + // ret = -1; } } else { DEBUG_WEBDAV("Error - put request on close failed: %d!", rc ); errno = EIO; ret = -1; } + /* Remove the local file. */ + _tunlink(tmpFileName); } c_free_multibyte(tmpFileName); } else { @@ -1320,9 +1568,9 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) { rc = ne_request_dispatch( writeCtx->req ); if( rc == NE_OK ) { if ( ne_get_status( writeCtx->req )->klass != 2 ) { - DEBUG_WEBDAV("Error - PUT status value no 2xx"); - errno = EIO; - ret = -1; + // DEBUG_WEBDAV("Error - PUT status value no 2xx"); + // errno = EIO; + // ret = -1; } } else { DEBUG_WEBDAV("Error - put request from memory failed: %d!", rc ); @@ -1331,6 +1579,7 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) { } } } + ne_request_destroy( writeCtx->req ); } else { /* Its a GET request, not much to do in close. */ @@ -1341,11 +1590,10 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) { } } } - /* Remove the local file. */ - unlink( writeCtx->tmpFileName ); /* free mem. Note that the request mem is freed by the ne_request_destroy call */ SAFE_FREE( writeCtx->tmpFileName ); + SAFE_FREE( writeCtx->clean_uri ); SAFE_FREE( writeCtx ); return ret; @@ -1481,11 +1729,7 @@ static csync_vio_file_stat_t *owncloud_readdir(csync_vio_method_handle_t *dhandl fetchCtx->currResource = fetchCtx->currResource->next; /* fill the static stat buf as input for the stat function */ - _fs.name = lfs->name; - _fs.mtime = lfs->mtime; - _fs.fields = lfs->fields; - _fs.type = lfs->type; - _fs.size = lfs->size; + fill_stat_cache(lfs); } /* DEBUG_WEBDAV("LFS fields: %s: %d", lfs->name, lfs->type ); */ @@ -1651,11 +1895,11 @@ static int owncloud_utimes(const char *uri, const struct timeval *times) { pname.name = "lastmodified"; newmodtime = modtime->tv_sec; -#if TIMEDELTA + DEBUG_WEBDAV("Add a time delta to modtime %lu: %ld", modtime->tv_sec, dav_session.time_delta); newmodtime += dav_session.time_delta; -#endif + snprintf( val, sizeof(val), "%ld", newmodtime ); DEBUG_WEBDAV("Setting LastModified of %s to %s", curi, val ); @@ -1678,7 +1922,8 @@ static int owncloud_utimes(const char *uri, const struct timeval *times) { csync_vio_method_t _method = { .method_table_size = sizeof(csync_vio_method_t), - .get_capabilities = owncloud_get_capabilities, + .get_capabilities = owncloud_capabilities, + .get_file_id = owncloud_file_id, .open = owncloud_open, .creat = owncloud_creat, .close = owncloud_close, @@ -1744,6 +1989,9 @@ void vio_module_shutdown(csync_vio_method_t *method) { SAFE_FREE( dav_session.proxy_user ); SAFE_FREE( dav_session.proxy_pwd ); + /* free stat memory */ + fill_stat_cache(NULL); + if( dav_session.ctx ) ne_session_destroy( dav_session.ctx ); /* DEBUG_WEBDAV( "********** vio_module_shutdown" ); */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e723badc2b..7974cbd82c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,11 +4,6 @@ add_subdirectory(std) find_package(SQLite3 REQUIRED) find_package(Iniparser REQUIRED) -if(CMAKE_CROSSCOMPILING) - find_package(OpenSSLCross REQUIRED) -else() - find_package(OpenSSL REQUIRED) -endif() set(CSYNC_PUBLIC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} @@ -17,7 +12,6 @@ set(CSYNC_PUBLIC_INCLUDE_DIRS ) set(CSYNC_PRIVATE_INCLUDE_DIRS - ${OPENSSL_INCLUDE_DIRS} ${INIPARSER_INCLUDE_DIRS} ${SQLITE3_INCLUDE_DIRS} ${CSTDLIB_PUBLIC_INCLUDE_DIRS} @@ -34,7 +28,6 @@ set(CSYNC_LINK_LIBRARIES ${CSTDLIB_LIBRARY} ${CSYNC_REQUIRED_LIBRARIES} ${INIPARSER_LIBRARIES} - ${OPENSSL_LIBRARIES} ${SQLITE3_LIBRARIES} ) @@ -48,6 +41,7 @@ set(csync_SRCS csync_config.c csync_exclude.c csync_statedb.c + csync_dbtree.c csync_time.c csync_util.c csync_misc.c diff --git a/src/csync.c b/src/csync.c index 9a58fb13a4..82e2468471 100644 --- a/src/csync.c +++ b/src/csync.c @@ -733,8 +733,10 @@ int csync_destroy(CSYNC *ctx) { /* free memory */ c_rbtree_free(ctx->local.tree); c_list_free(ctx->local.list); + c_list_free(ctx->local.id_list); c_rbtree_free(ctx->remote.tree); c_list_free(ctx->remote.list); + c_list_free(ctx->remote.id_list); SAFE_FREE(ctx->local.uri); SAFE_FREE(ctx->remote.uri); SAFE_FREE(ctx->options.config_dir); diff --git a/src/csync.h b/src/csync.h index 05c6c01af7..b51ce9e4dc 100644 --- a/src/csync.h +++ b/src/csync.h @@ -51,7 +51,7 @@ extern "C" { /* csync version */ #define LIBCSYNC_VERSION_MAJOR 0 #define LIBCSYNC_VERSION_MINOR 50 -#define LIBCSYNC_VERSION_MICRO 8 +#define LIBCSYNC_VERSION_MICRO 10 #define LIBCSYNC_VERSION_INT CSYNC_VERSION_INT(LIBCSYNC_VERSION_MAJOR, \ LIBCSYNC_VERSION_MINOR, \ diff --git a/src/csync_dbtree.c b/src/csync_dbtree.c new file mode 100644 index 0000000000..462f9c1f2f --- /dev/null +++ b/src/csync_dbtree.c @@ -0,0 +1,231 @@ +/* + * libcsync -- a library to sync a directory with another + * + * Copyright (c) 2012 by Klaas Freitag + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include "config.h" + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "csync_dbtree.h" +#include "c_lib.h" +#include "csync_private.h" +#include "csync_statedb.h" +#include "csync_util.h" + + +#define CSYNC_LOG_CATEGORY_NAME "csync.dbtree" +#include "csync_log.h" + +struct dir_listing { + c_list_t *list; + unsigned int cnt; + c_list_t *entry; + char *dir; +}; + +csync_vio_method_handle_t *csync_dbtree_opendir(CSYNC *ctx, const char *name) +{ + char *stmt = NULL; + char *column = NULL; + const char *path = NULL; + csync_vio_file_stat_t *fs = NULL; + unsigned int c = 0; + c_strlist_t *list = NULL; + struct dir_listing *listing = NULL; + + /* "phash INTEGER(8)," + "pathlen INTEGER," + "path VARCHAR(4096)," + "inode INTEGER," + "uid INTEGER," + "gid INTEGER," + "mode INTEGER," + "modtime INTEGER(8)," + "type INTEGER," + "md5 VARCHAR(32)," + */ + + int col_count = 9; + if( strlen(name) < strlen(ctx->remote.uri)+1) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "name does not contain remote uri!"); + return NULL; + } + + path = name + strlen(ctx->remote.uri)+1; + + if( asprintf( &stmt, "SELECT phash, path, inode, uid, gid, mode, modtime, type, md5 " + "FROM metadata WHERE path GLOB('%s/*')", path ) < 0 ) { + return NULL; + } + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SQL: %s", stmt); + + list = csync_statedb_query( ctx, stmt ); + + if( ! list ) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Query result list is NULL!"); + return NULL; + } + /* list count must be a multiple of col_count */ + if( list->count % col_count != 0 ) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Wrong size of query result list"); + return NULL; + } + + listing = c_malloc(sizeof(struct dir_listing)); + if( listing == NULL ) { + errno = ENOMEM; + return NULL; + } + + listing->dir = c_strdup(path); + + for( c = 0; c < (list->count / col_count); c++) { + int base = c*col_count; + int cnt = 0; + int tpath_len = 0; + int type = 0; + + char *tpath = list->vector[base+1]; + /* check if the result points to a file directly below the search path + * by checking if there is another / in the result. + * If yes, skip it. + * FIXME: Find a better filter solution here. + */ + tpath += strlen(path)+1; /* jump over the search path */ + tpath_len = strlen( tpath ); + while( cnt < tpath_len ) { + + if(*(tpath+cnt) == '/') { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Skipping entry: %s", list->vector[base+1]); + break; + } + cnt++; + } + if( cnt < tpath_len ) continue; + + fs = csync_vio_file_stat_new(); + fs->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE; + + column = list->vector[base+0]; /* phash */ + + column = list->vector[base+1]; /* path */ + fs->name = c_strdup(column+strlen(path)+1); + + column = list->vector[base+2]; /* inode */ + fs->inode = atoi(column); + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_INODE; + + column = list->vector[base+3]; /* uid */ + fs->uid = atoi(column); + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_UID; + + column = list->vector[base+4]; /* gid */ + fs->gid = atoi(column); + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_GID; + + column = list->vector[base+5]; /* mode */ + fs->mode = atoi(column); + // fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_M; + + column = list->vector[base+6]; /* modtime */ + fs->mtime = strtoul(column, NULL, 10); + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME; + + column = list->vector[base+7]; /* type */ + type = atoi(column); + /* Attention: the type of csync_ftw_type_e which is the source for + * the database entry is different from csync_vio_file_type_e which + * is the target file type here. Mapping is needed! + */ + switch( type ) { + case CSYNC_FTW_TYPE_DIR: + fs->type = CSYNC_VIO_FILE_TYPE_DIRECTORY; + break; + case CSYNC_FTW_TYPE_FILE: + fs->type = CSYNC_VIO_FILE_TYPE_REGULAR; + break; + case CSYNC_FTW_TYPE_SLINK: + fs->type = CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK; + break; + default: + fs->type = CSYNC_VIO_FILE_TYPE_UNKNOWN; + } + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE; + + column = list->vector[base+8]; /* type */ + fs->md5 = c_strdup(column); + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; + + /* store into result list. */ + listing->list = c_list_append( listing->list, fs ); + listing->cnt++; + } + listing->entry = c_list_first( listing->list ); + + c_strlist_destroy( list ); + SAFE_FREE(stmt); + + return listing; +} + +int csync_dbtree_closedir(CSYNC *ctx, csync_vio_method_handle_t *dhandle) +{ + struct dir_listing *dl = NULL; + int rc = 0; + (void) ctx; + + if( dhandle != NULL ) { + dl = (struct dir_listing*) dhandle; + c_list_free(dl->list); + SAFE_FREE(dl->dir); + SAFE_FREE(dl); + } + + return rc; +} + +csync_vio_file_stat_t *csync_dbtree_readdir(CSYNC *ctx, csync_vio_method_handle_t *dhandle) +{ + csync_vio_file_stat_t *fs = NULL; + struct dir_listing *dl = NULL; + (void) ctx; + + if( dhandle != NULL ) { + dl = (struct dir_listing*) dhandle; + if( dl->entry != NULL ) { + fs = (csync_vio_file_stat_t*) dl->entry->data; + + dl->entry = c_list_next( dl->entry); + } + } + + return fs; +} + +/* vim: set ts=8 sw=2 et cindent: */ diff --git a/src/csync_dbtree.h b/src/csync_dbtree.h new file mode 100644 index 0000000000..297a9995f0 --- /dev/null +++ b/src/csync_dbtree.h @@ -0,0 +1,60 @@ +/* + * libcsync -- a library to sync a directory with another + * + * Copyright (c) 2012 by Klaas Freitag + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file csync_dbtree.h + * + * @brief Private interface of csync + * + * @defgroup csyncdbtreeInternals csync statedb internals + * @ingroup csyncInternalAPI + * + * @{ + */ + +#ifndef _CSYNC_DBTREE_H +#define _CSYNC_DBTREE_H + +#include "c_lib.h" +#include "csync_private.h" +#include "vio/csync_vio_handle.h" + +/** + * @brief Open a directory based on the statedb. + * + * This function reads the list of files within a directory from statedb and + * builds up a list in memory. + * + * @param ctx The csync context. + * @param name The directory name. + * + * @return 0 on success, less than 0 if an error occured with errno set. + */ +csync_vio_method_handle_t *csync_dbtree_opendir(CSYNC *ctx, const char *name); + +int csync_dbtree_closedir(CSYNC *ctx, csync_vio_method_handle_t *dhandle); + +csync_vio_file_stat_t *csync_dbtree_readdir(CSYNC *ctx, csync_vio_method_handle_t *dhandle); + +/** + * }@ + */ +#endif /* _CSYNC_DBTREE_H */ +/* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */ diff --git a/src/csync_exclude.c b/src/csync_exclude.c index 3ffe93eb07..0f7163c8a7 100644 --- a/src/csync_exclude.c +++ b/src/csync_exclude.c @@ -109,6 +109,7 @@ void csync_exclude_destroy(CSYNC *ctx) { int csync_excluded(CSYNC *ctx, const char *path) { size_t i; const char *p; + const char *bname = NULL; if (! ctx->options.unix_extensions) { for (p = path; *p; p++) { @@ -133,11 +134,16 @@ int csync_excluded(CSYNC *ctx, const char *path) { } if (ctx->excludes->count) { + bname = c_basename(path); for (i = 0; i < ctx->excludes->count; i++) { if (csync_fnmatch(ctx->excludes->vector[i], path, 0) == 0) { return 1; } + if( bname && csync_fnmatch(ctx->excludes->vector[i], bname, 0) == 0) { + return 1; + } } + SAFE_FREE(bname); } return 0; } diff --git a/src/csync_lock.c b/src/csync_lock.c index cec397c01a..ef58c8fe15 100644 --- a/src/csync_lock.c +++ b/src/csync_lock.c @@ -63,6 +63,7 @@ static int _csync_lock_create(const char *lockfile) { goto out; } + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Create temporary lock file: %s", ctmpfile); if ((fd = mkstemp(ctmpfile)) < 0) { strerror_r(errno, errbuf, sizeof(errbuf)); diff --git a/src/csync_private.h b/src/csync_private.h index a10e40a1c1..51eb987f73 100644 --- a/src/csync_private.h +++ b/src/csync_private.h @@ -94,6 +94,7 @@ struct csync_s { char *uri; c_rbtree_t *tree; c_list_t *list; + c_list_t *id_list; enum csync_replica_e type; } local; @@ -101,7 +102,9 @@ struct csync_s { char *uri; c_rbtree_t *tree; c_list_t *list; + c_list_t *id_list; enum csync_replica_e type; + int read_from_db; } remote; struct { @@ -135,7 +138,7 @@ struct csync_s { /* error code of the last operation */ enum csync_error_codes_e error_code; - + int status; }; @@ -159,7 +162,10 @@ struct csync_file_stat_s { mode_t mode; /* u32 */ int nlink; /* u32 */ int type; /* u32 */ + char *destpath; /* for renames */ + const char *md5; + enum csync_instructions_e instruction; /* u32 */ char path[1]; /* u8 */ } diff --git a/src/csync_propagate.c b/src/csync_propagate.c index fa59519f0c..34ace8740d 100644 --- a/src/csync_propagate.c +++ b/src/csync_propagate.c @@ -28,10 +28,12 @@ #include #include #include +#include #include "csync_private.h" #include "csync_propagate.h" #include "vio/csync_vio.h" +#include "c_jhash.h" #define CSYNC_LOG_CATEGORY_NAME "csync.propagator" #include "csync_log.h" @@ -46,6 +48,27 @@ static int _csync_cleanup_cmp(const void *a, const void *b) { return strcmp(st_a->path, st_b->path); } +static void _store_id_update(CSYNC *ctx, csync_file_stat_t *st) { + c_list_t *list = NULL; + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SYNCED remember dir: %s", st->path); + + switch (ctx->current) { + case LOCAL_REPLICA: + list = c_list_prepend(ctx->local.id_list, (void*)st); + if( list != NULL ) { + ctx->local.id_list = list; + } + break; + case REMOTE_REPLCIA: + list = c_list_prepend(ctx->remote.id_list, (void*)st); + if(list != NULL ) { + ctx->remote.id_list = list; + } + break; + + } +} + static bool _push_to_tmp_first(CSYNC *ctx) { if( ctx->current == REMOTE_REPLCIA ) return true; /* Always push to tmp for destination local file system */ @@ -56,6 +79,22 @@ static bool _push_to_tmp_first(CSYNC *ctx) return false; } +static const char*_get_md5( CSYNC *ctx, const char *path ) { + const char *md5 = NULL; + char *buf = NULL; + + /* Always use the remote uri path, local does not have Ids. */ + if (asprintf(&buf, "%s/%s", ctx->remote.uri, path) < 0) { + return 0; + } + + md5 = csync_vio_file_id(ctx, buf); + + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "MD5 for %s: %s", buf, md5 ? md5 : ""); + SAFE_FREE(buf); + return md5; +} + static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) { enum csync_replica_e srep = -1; enum csync_replica_e drep = -1; @@ -65,6 +104,7 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) { char *duri = NULL; char *turi = NULL; char *tdir = NULL; + const char *tmd5 = NULL; csync_vio_handle_t *sfp = NULL; csync_vio_handle_t *dfp = NULL; @@ -332,6 +372,18 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) { rc = 1; goto out; } + + if( st->md5 ) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "UUUU MD5 sum: %s", st->md5); + } else { + if( tstat->md5 ) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Target MD5 sum is %s", tstat->md5 ); + if(st->md5) SAFE_FREE(st->md5); + st->md5 = c_strdup(tstat->md5 ); + } else { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "MD5 sum is empty"); + } + } } if (_push_to_tmp_first(ctx)) { @@ -386,6 +438,17 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) { ctx->replica = drep; csync_vio_utimes(ctx, duri, times); + + /* For remote repos, after the utimes call, the ID has changed again */ + /* do a stat on the target again to get a valid md5 */ + tmd5 = _get_md5(ctx, st->path); + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "FINAL MD5: %s", tmd5 ? tmd5 : ""); + + if(tmd5) { + SAFE_FREE(st->md5); + st->md5 = tmd5; + } + /* set instruction for the statedb merger */ st->instruction = CSYNC_INSTRUCTION_UPDATED; @@ -556,6 +619,8 @@ static int _csync_rename_file(CSYNC *ctx, csync_file_stat_t *st) { struct timeval times[2]; char *suri = NULL; char *duri = NULL; + const char *tmd5 = NULL; + c_rbnode_t *node = NULL; switch (ctx->current) { case REMOTE_REPLCIA: @@ -605,16 +670,36 @@ static int _csync_rename_file(CSYNC *ctx, csync_file_stat_t *st) { csync_vio_utimes(ctx, duri, times); - /* set instruction for the statedb merger */ + /* The the uniq ID for the destination */ + tmd5 = _get_md5(ctx, st->destpath); + if( rc > -1 ) { - st->instruction = CSYNC_INSTRUCTION_RENAME; - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "RENAME file: %s", suri); + /* Find the destination entry in the local tree and insert the uniq id */ + int len = strlen(st->destpath); + uint64_t h = c_jhash64((uint8_t *) st->destpath, len, 0); + h = c_jhash64((uint8_t *) st->destpath, len, 0); + + /* search in the local tree for the local file to get the mtime */ + node = c_rbtree_find(ctx->local.tree, &h); + if(node == NULL) { + /* no local file found. */ + + } else { + csync_file_stat_t *other = NULL; + /* set the mtime which is needed in statedb_get_uniqid */ + other = (csync_file_stat_t *) node->data; + if( other ) { + other->md5 = tmd5; + } + } + /* set instruction for the statedb merger */ + st->instruction = CSYNC_INSTRUCTION_DELETED; } + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "RENAME file: %s => %s with ID %s", st->path, st->destpath, st->md5); out: SAFE_FREE(suri); SAFE_FREE(duri); - SAFE_FREE(st->destpath); /* set instruction for the statedb merger */ if (rc != 0) { @@ -705,6 +790,7 @@ static int _csync_new_dir(CSYNC *ctx, csync_file_stat_t *st) { enum csync_replica_e replica_bak; char errbuf[256] = {0}; char *uri = NULL; + const char *tmd5 = NULL; struct timeval times[2]; int rc = -1; @@ -775,6 +861,12 @@ static int _csync_new_dir(CSYNC *ctx, csync_file_stat_t *st) { csync_vio_utimes(ctx, uri, times); + tmd5 = _get_md5(ctx, st->path); + if(tmd5) { + SAFE_FREE(st->md5); + st->md5 = tmd5; + } + /* set instruction for the statedb merger */ st->instruction = CSYNC_INSTRUCTION_UPDATED; @@ -798,6 +890,7 @@ static int _csync_sync_dir(CSYNC *ctx, csync_file_stat_t *st) { enum csync_replica_e replica_bak; char errbuf[256] = {0}; char *uri = NULL; + const char *tmd5 = NULL; struct timeval times[2]; int rc = -1; @@ -851,11 +944,16 @@ static int _csync_sync_dir(CSYNC *ctx, csync_file_stat_t *st) { times[0].tv_usec = times[1].tv_usec = 0; csync_vio_utimes(ctx, uri, times); - + tmd5 = _get_md5(ctx, st->path); + if(tmd5) { + SAFE_FREE(st->md5); + st->md5 = tmd5; + } /* set instruction for the statedb merger */ st->instruction = CSYNC_INSTRUCTION_UPDATED; CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SYNCED dir: %s", uri); + ctx->replica = replica_bak; rc = 0; @@ -951,6 +1049,148 @@ out: return rc; } +static int _cmp_char( const void *d1, const void *d2 ) +{ + const char *c1 = (const char*) d1; + const char *c2 = (const char*) d2; + // CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "COMPARE: %s <-> %s", c1, c2); + if( c_streq(c1, c2) ) return 0; + return 1; +} + +/* + * This function corrects the unique IDs of parent directories of changed + * files. Other than in the file system, the change of a unique Id propagates + * up to the top dir. To save the correct IDs, in all propagations, pathes + * are recorded in the local and remotes id_list lists. + * In this function, the unique ID is queried for each directory once and + * stored into the according entry. + */ +static int _csync_correct_id(CSYNC *ctx) { + c_list_t *walk = NULL; + c_list_t *seen_dirs = NULL; + c_list_t *list = NULL; + c_rbtree_t *tree = NULL; + const char *replica = NULL; + char *path = NULL; + + switch (ctx->current) { + case LOCAL_REPLICA: + list = ctx->local.id_list; + tree = ctx->local.tree; + replica = "LOCAL_REPLICA"; + break; + case REMOTE_REPLCIA: + list = ctx->remote.id_list; + tree = ctx->remote.tree; + replica = "REMOTE_REPLICA"; + break; + default: + break; + } + + if (list == NULL) { + return 0; + } + + list = c_list_sort(list, _csync_cleanup_cmp); + if (list == NULL) { + return -1; + } + + for (walk = c_list_last(list); walk != NULL; walk = c_list_prev(walk)) { + csync_file_stat_t *st = NULL; + + st = (csync_file_stat_t *) walk->data; + if( st->type == CSYNC_FTW_TYPE_FILE ) { + path = c_dirname( st->path ); + } else if( st->type == CSYNC_FTW_TYPE_DIR ) { + path = c_strdup(st->path); /* Allocate mem for this not to disappoint FREE */ + } else { + /* unhandled path */ + } + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "correct ID on dir: %s", path); + + /* handle the . path */ + if( path && path[0] == '.' && strlen(path) == 1) { + SAFE_FREE(path); + path = NULL; + } + + while( path ) { + uint64_t h; + int len; + c_rbnode_t *node = NULL; + + char pathbuf[PATH_MAX] = {'\0' }; + char *old_path = path; + csync_file_stat_t *tfs = NULL; + + /* do stuff with the dir here */ + + if( seen_dirs && c_list_find_custom( seen_dirs, path, _cmp_char)) { + // CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "saw this dir already: %s", path); + } else { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "climb on dir: %s (%s)", path, replica); + seen_dirs = c_list_prepend( seen_dirs, c_strdup(path)); + + /* Find the correct target entry. */ + len = strlen(path); + h = c_jhash64((uint8_t *) path, len, 0); + + node = c_rbtree_find(tree, &h); + if (node == NULL) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Unable to find node"); + } else { + tfs = c_rbtree_node_data(node); + if( tfs ) { + if(tfs->instruction == CSYNC_INSTRUCTION_DELETED) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Skipping update of MD5 because item is deleted."); + } else { + if(tfs->md5) SAFE_FREE(tfs->md5); + tfs->md5 = _get_md5(ctx, path); + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "MD5 for dir: %s %s (Instruction: %s)", tfs->path, + tfs->md5, csync_instruction_str(tfs->instruction)); + if( tfs->md5 && tfs->instruction == CSYNC_INSTRUCTION_NONE ) { + /* set instruction for the statedb merger */ + tfs->instruction = CSYNC_INSTRUCTION_UPDATED; + } + } + } + } + } + /* get the parent dir */ + /* copy one byte less, omit the trailing slash */ + strncpy( pathbuf, path, strlen(path)-1 ); + /* and get the path name */ + path = c_dirname( pathbuf ); + /* free the old path memory */ + SAFE_FREE(old_path ); + + /* exit on top directory */ + if( c_streq(path, ".")) { + SAFE_FREE(path); + path = NULL; + } + } + } + + /* Free the seen_dirs list */ + + if( seen_dirs ) { + c_list_t *walk1 = NULL; + + for (walk1 = c_list_first(seen_dirs); walk1 != NULL; walk1 = c_list_next(walk1)) { + char *data = NULL; + + data = (char*) walk1->data; + SAFE_FREE(data); + } + } + c_list_free(seen_dirs); + return 0; +} + static int _csync_propagation_cleanup(CSYNC *ctx) { c_list_t *list = NULL; c_list_t *walk = NULL; @@ -1020,30 +1260,35 @@ static int _csync_propagation_file_visitor(void *obj, void *data) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL NEW: %s",st->path); goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_RENAME: if (_csync_rename_file(ctx, st) < 0) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL RENAME: %s",st->path); goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_SYNC: if (_csync_sync_file(ctx, st) < 0) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL SYNC: %s",st->path); goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_REMOVE: if (_csync_remove_file(ctx, st) < 0) { CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL REMOVE: %s",st->path); goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_CONFLICT: CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"case CSYNC_INSTRUCTION_CONFLICT: %s",st->path); if (_csync_conflict_file(ctx, st) < 0) { goto err; } + _store_id_update(ctx, st); break; default: break; @@ -1084,22 +1329,27 @@ static int _csync_propagation_dir_visitor(void *obj, void *data) { if (_csync_new_dir(ctx, st) < 0) { goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_SYNC: if (_csync_sync_dir(ctx, st) < 0) { goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_CONFLICT: CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"directory attributes different"); if (_csync_sync_dir(ctx, st) < 0) { goto err; } + _store_id_update(ctx, st); break; case CSYNC_INSTRUCTION_REMOVE: if (_csync_remove_dir(ctx, st) < 0) { goto err; } + _store_id_update(ctx, st); + break; case CSYNC_INSTRUCTION_RENAME: /* there can't be a rename for dirs. See updater. */ break; @@ -1142,6 +1392,9 @@ int csync_propagate_files(CSYNC *ctx) { return -1; } + if( _csync_correct_id(ctx) < 0) { + return -1; + } return 0; } diff --git a/src/csync_reconcile.c b/src/csync_reconcile.c index 93fb88bce8..97a78cdaad 100644 --- a/src/csync_reconcile.c +++ b/src/csync_reconcile.c @@ -43,269 +43,272 @@ * source and the destination, have been changed, the newer file wins. */ static int _csync_merge_algorithm_visitor(void *obj, void *data) { - csync_file_stat_t *cur = NULL; - csync_file_stat_t *other = NULL; - csync_file_stat_t *tmp = NULL; - uint64_t h = 0; - int len = 0; + csync_file_stat_t *cur = NULL; + csync_file_stat_t *other = NULL; + csync_file_stat_t *tmp = NULL; + uint64_t h = 0; + int len = 0; - CSYNC *ctx = NULL; - c_rbtree_t *tree = NULL; - c_rbnode_t *node = NULL; + CSYNC *ctx = NULL; + c_rbtree_t *tree = NULL; + c_rbnode_t *node = NULL; - cur = (csync_file_stat_t *) obj; - ctx = (CSYNC *) data; + cur = (csync_file_stat_t *) obj; + ctx = (CSYNC *) data; - /* we need the opposite tree! */ - switch (ctx->current) { + /* we need the opposite tree! */ + switch (ctx->current) { case LOCAL_REPLICA: - tree = ctx->remote.tree; - break; + tree = ctx->remote.tree; + break; case REMOTE_REPLCIA: - tree = ctx->local.tree; - break; + tree = ctx->local.tree; + break; default: - break; - } - - node = c_rbtree_find(tree, &cur->phash); - /* file only found on current replica */ - if (node == NULL) { - switch(cur->instruction) { - /* file has been modified */ - case CSYNC_INSTRUCTION_EVAL: - cur->instruction = CSYNC_INSTRUCTION_NEW; - break; - /* file has been removed on the opposite replica */ - case CSYNC_INSTRUCTION_NONE: - cur->instruction = CSYNC_INSTRUCTION_REMOVE; - break; - case CSYNC_INSTRUCTION_RENAME: - /* rename support only on the local replica because of inode needed. */ - if(ctx->current == LOCAL_REPLICA ) { - /* use the old name to find the "other" node */ - tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); - /* Find the opposite node. */ - if( tmp ) { - /* We need to calculate the phash again because of the phash being stored as int in db. */ - if( tmp->path ) { - len = strlen( tmp->path ); - h = c_jhash64((uint8_t *) tmp->path, len, 0); - - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"PHash of temporar opposite: %llu", h); - node = c_rbtree_find(tree, &h); - } - if(node) { - other = (csync_file_stat_t*)node->data; - other->instruction = CSYNC_INSTRUCTION_RENAME; - other->destpath = c_strdup( cur->path ); - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - if( ! other ) { - cur->instruction = CSYNC_INSTRUCTION_NEW; - } - } - } - break; - default: break; } - } else { - /* + + node = c_rbtree_find(tree, &cur->phash); + /* file only found on current replica */ + if (node == NULL) { + switch(cur->instruction) { + /* file has been modified */ + case CSYNC_INSTRUCTION_EVAL: + cur->instruction = CSYNC_INSTRUCTION_NEW; + break; + /* file has been removed on the opposite replica */ + case CSYNC_INSTRUCTION_NONE: + cur->instruction = CSYNC_INSTRUCTION_REMOVE; + break; + case CSYNC_INSTRUCTION_RENAME: + /* rename support only on the local replica because of inode needed. */ + if(ctx->current == LOCAL_REPLICA ) { + /* use the old name to find the "other" node */ + tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); + /* Find the opposite node. */ + if( tmp ) { + /* We need to calculate the phash again because of the phash being stored as int in db. */ + if( tmp->path ) { + len = strlen( tmp->path ); + h = c_jhash64((uint8_t *) tmp->path, len, 0); + + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"PHash of temporar opposite: %llu", h); + node = c_rbtree_find(tree, &h); + } + if(node) { + other = (csync_file_stat_t*)node->data; + other->instruction = CSYNC_INSTRUCTION_RENAME; + other->destpath = c_strdup( cur->path ); + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + if( ! other ) { + cur->instruction = CSYNC_INSTRUCTION_NEW; + } + } + } + break; + default: + break; + } + } else { + /* * file found on the other replica */ - other = (csync_file_stat_t *) node->data; + other = (csync_file_stat_t *) node->data; - switch (cur->instruction) { - /* file on current replica is new */ - case CSYNC_INSTRUCTION_NEW: - switch (other->instruction) { - /* file on other replica is new too */ - case CSYNC_INSTRUCTION_NEW: - if (cur->modtime > other->modtime) { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, cur is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_CONFLICT; - other->instruction = CSYNC_INSTRUCTION_NONE; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_SYNC; - other->instruction = CSYNC_INSTRUCTION_NONE; - } - - } else if (cur->modtime < other->modtime) { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, other is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_NONE; - other->instruction = CSYNC_INSTRUCTION_CONFLICT; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_NONE; - other->instruction = CSYNC_INSTRUCTION_SYNC; - } - - } else { - /* file are equal */ - cur->instruction = CSYNC_INSTRUCTION_NONE; - other->instruction = CSYNC_INSTRUCTION_NONE; + switch (cur->instruction) { + /* file on current replica is new */ + case CSYNC_INSTRUCTION_NEW: + switch (other->instruction) { + /* file on other replica is new too */ + case CSYNC_INSTRUCTION_NEW: + if (cur->modtime > other->modtime) { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, cur is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_CONFLICT; + other->instruction = CSYNC_INSTRUCTION_NONE; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_SYNC; + other->instruction = CSYNC_INSTRUCTION_NONE; + } + + } else if (cur->modtime < other->modtime) { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, other is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_NONE; + other->instruction = CSYNC_INSTRUCTION_CONFLICT; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_NONE; + other->instruction = CSYNC_INSTRUCTION_SYNC; + } + + } else { + /* file are equal */ + /* FIXME: Get the id from the server! */ + cur->instruction = CSYNC_INSTRUCTION_NONE; + other->instruction = CSYNC_INSTRUCTION_NONE; + + if( !cur->md5 ) cur->md5 = other->md5; + } + break; + /* file on other replica has changed too */ + case CSYNC_INSTRUCTION_EVAL: + /* file on current replica is newer */ + if (cur->modtime > other->modtime) { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, cur is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_CONFLICT; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_SYNC; + } + + } else { + /* file on opposite replica is newer */ + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, other is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + + } + break; + /* file on the other replica has not been modified */ + case CSYNC_INSTRUCTION_NONE: + cur->instruction = CSYNC_INSTRUCTION_SYNC; + break; + default: + break; } break; - /* file on other replica has changed too */ - case CSYNC_INSTRUCTION_EVAL: - /* file on current replica is newer */ - if (cur->modtime > other->modtime) { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, cur is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_CONFLICT; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_SYNC; - } - - } else { - /* file on opposite replica is newer */ - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, other is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - + /* file on current replica has been modified */ + case CSYNC_INSTRUCTION_EVAL: + switch (other->instruction) { + /* file on other replica is new too */ + case CSYNC_INSTRUCTION_NEW: + if (cur->modtime > other->modtime) { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, cur is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_CONFLICT; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_SYNC; + } + + } else { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, other is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + } + break; + /* file on other replica has changed too */ + case CSYNC_INSTRUCTION_EVAL: + /* file on current replica is newer */ + if (cur->modtime > other->modtime) { + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, cur is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_CONFLICT; + other->instruction= CSYNC_INSTRUCTION_NONE; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_SYNC; + } + + } else { + /* file on opposite replica is newer */ + + if(ctx->options.with_conflict_copys) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, other is newer PATH=./%s",cur->path); + cur->instruction = CSYNC_INSTRUCTION_NONE; + other->instruction=CSYNC_INSTRUCTION_CONFLICT; + } + else + { + cur->instruction = CSYNC_INSTRUCTION_NONE; + } + } + break; + /* file on the other replica has not been modified */ + case CSYNC_INSTRUCTION_NONE: + cur->instruction = CSYNC_INSTRUCTION_SYNC; + break; + default: + break; } break; - /* file on the other replica has not been modified */ - case CSYNC_INSTRUCTION_NONE: - cur->instruction = CSYNC_INSTRUCTION_SYNC; - break; - default: + default: break; } - break; - /* file on current replica has been modified */ - case CSYNC_INSTRUCTION_EVAL: - switch (other->instruction) { - /* file on other replica is new too */ - case CSYNC_INSTRUCTION_NEW: - if (cur->modtime > other->modtime) { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, cur is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_CONFLICT; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_SYNC; - } - - } else { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, other is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - } - break; - /* file on other replica has changed too */ - case CSYNC_INSTRUCTION_EVAL: - /* file on current replica is newer */ - if (cur->modtime > other->modtime) { - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, cur is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_CONFLICT; - other->instruction= CSYNC_INSTRUCTION_NONE; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_SYNC; - } - - } else { - /* file on opposite replica is newer */ - - if(ctx->options.with_conflict_copys) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, other is newer PATH=./%s",cur->path); - cur->instruction = CSYNC_INSTRUCTION_NONE; - other->instruction=CSYNC_INSTRUCTION_CONFLICT; - } - else - { - cur->instruction = CSYNC_INSTRUCTION_NONE; - } - } - break; - /* file on the other replica has not been modified */ - case CSYNC_INSTRUCTION_NONE: - cur->instruction = CSYNC_INSTRUCTION_SYNC; - break; - default: - break; - } - break; - default: - break; } - } - - //hide instruction NONE messages when log level is set to debug, - //only show these messages on log level trace - if(cur->instruction ==CSYNC_INSTRUCTION_NONE) - { - if(cur->type == CSYNC_FTW_TYPE_DIR) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, - "%-20s dir: %s", - csync_instruction_str(cur->instruction), - cur->path); - } - else - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, - "%-20s file: %s", - csync_instruction_str(cur->instruction), - cur->path); - } - } - else - { - if(cur->type == CSYNC_FTW_TYPE_DIR) - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, - "%-20s dir: %s", - csync_instruction_str(cur->instruction), - cur->path); - } - else - { - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, - "%-20s file: %s", - csync_instruction_str(cur->instruction), - cur->path); - } - } - - return 0; + + //hide instruction NONE messages when log level is set to debug, + //only show these messages on log level trace + if(cur->instruction ==CSYNC_INSTRUCTION_NONE) + { + if(cur->type == CSYNC_FTW_TYPE_DIR) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, + "%-20s dir: %s", + csync_instruction_str(cur->instruction), + cur->path); + } + else + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, + "%-20s file: %s", + csync_instruction_str(cur->instruction), + cur->path); + } + } + else + { + if(cur->type == CSYNC_FTW_TYPE_DIR) + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, + "%-20s dir: %s", + csync_instruction_str(cur->instruction), + cur->path); + } + else + { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, + "%-20s file: %s", + csync_instruction_str(cur->instruction), + cur->path); + } + } + + return 0; } int csync_reconcile_updates(CSYNC *ctx) { diff --git a/src/csync_statedb.c b/src/csync_statedb.c index feb85a9596..e51e1abab0 100644 --- a/src/csync_statedb.c +++ b/src/csync_statedb.c @@ -213,6 +213,8 @@ int csync_statedb_create_tables(CSYNC *ctx) { "gid INTEGER," "mode INTEGER," "modtime INTEGER(8)," + "type INTEGER," + "md5 VARCHAR(32)," "PRIMARY KEY(phash)" ");" ); @@ -232,6 +234,8 @@ int csync_statedb_create_tables(CSYNC *ctx) { "gid INTEGER," "mode INTEGER," "modtime INTEGER(8)," + "type INTEGER," + "md5 VARCHAR(32)," "PRIMARY KEY(phash)" ");" ); @@ -254,6 +258,13 @@ int csync_statedb_create_tables(CSYNC *ctx) { } c_strlist_destroy(result); + result = csync_statedb_query(ctx, + "CREATE INDEX metadata_md5 ON metadata(md5);"); + if (result == NULL) { + return -1; + } + c_strlist_destroy(result); + return 0; } @@ -296,8 +307,8 @@ static int _insert_metadata_visitor(void *obj, void *data) { case CSYNC_INSTRUCTION_CONFLICT: CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "SQL statement: INSERT INTO metadata_temp \n" - "\t\t\t(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES \n" - "\t\t\t(%llu, %lu, %s, %llu, %u, %u, %u, %lu);", + "\t\t\t(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES \n" + "\t\t\t(%llu, %lu, %s, %llu, %u, %u, %u, %lu, %d, %s);", (long long unsigned int) fs->phash, (long unsigned int) fs->pathlen, fs->path, @@ -305,14 +316,16 @@ static int _insert_metadata_visitor(void *obj, void *data) { fs->uid, fs->gid, fs->mode, - fs->modtime); + fs->modtime, + fs->type, + fs->md5 ? fs->md5 : ""); /* * The phash needs to be long long unsigned int or it segfaults on PPC */ stmt = sqlite3_mprintf("INSERT INTO metadata_temp " - "(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES " - "(%llu, %lu, '%q', %llu, %u, %u, %u, %lu);", + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES " + "(%llu, %lu, '%q', %llu, %u, %u, %u, %lu, %d, '%s');", (long long unsigned int) fs->phash, (long unsigned int) fs->pathlen, fs->path, @@ -320,7 +333,9 @@ static int _insert_metadata_visitor(void *obj, void *data) { fs->uid, fs->gid, fs->mode, - fs->modtime); + fs->modtime, + fs->type, + fs->md5); if (stmt == NULL) { return -1; @@ -381,19 +396,19 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash) { return NULL; } - if (result->count <= 6) { - c_strlist_destroy(result); - return NULL; - } - /* phash, pathlen, path, inode, uid, gid, mode, modtime */ - len = strlen(result->vector[2]); - st = c_malloc(sizeof(csync_file_stat_t) + len + 1); - if (st == NULL) { - c_strlist_destroy(result); - return NULL; + if (result->count != 0 && result->count < 10) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Amount of result columns wrong, db version mismatch!"); } + if(result->count > 7) { + /* phash, pathlen, path, inode, uid, gid, mode, modtime */ + len = strlen(result->vector[2]); + st = c_malloc(sizeof(csync_file_stat_t) + len + 1); + if (st == NULL) { + c_strlist_destroy(result); + return NULL; + } - /* + /* * FIXME: * We use an INTEGER(8) which is signed to the phash in the sqlite3 db, * but the phash is an uint64_t. So for some values we get a string like @@ -403,17 +418,28 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash) { * st->phash = strtoull(result->vector[0], NULL, 10); */ - /* The query suceeded so use the phash we pass to the function. */ - st->phash = phash; + /* The query suceeded so use the phash we pass to the function. */ + st->phash = phash; - st->pathlen = atoi(result->vector[1]); - memcpy(st->path, (len ? result->vector[2] : ""), len + 1); - st->inode = atoi(result->vector[3]); - st->uid = atoi(result->vector[4]); - st->gid = atoi(result->vector[5]); - st->mode = atoi(result->vector[6]); - st->modtime = strtoul(result->vector[7], NULL, 10); + st->pathlen = atoi(result->vector[1]); + memcpy(st->path, (len ? result->vector[2] : ""), len + 1); + st->inode = atoi(result->vector[3]); + st->uid = atoi(result->vector[4]); + st->gid = atoi(result->vector[5]); + st->mode = atoi(result->vector[6]); + st->modtime = strtoul(result->vector[7], NULL, 10); + } else { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "No result record found for phash = %llu", + (long long unsigned int) phash); + } + if(result->count > 8 && result->vector[8]) { + st->type = atoi(result->vector[8]); + } + + if(result->count > 9 && result->vector[9]) { + st->md5 = c_strdup( result->vector[9] ); + } c_strlist_destroy(result); return st; @@ -458,12 +484,38 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode) { st->gid = atoi(result->vector[5]); st->mode = atoi(result->vector[6]); st->modtime = strtoul(result->vector[7], NULL, 10); + st->type = atoi(result->vector[8]); + if( result->vector[9] ) + st->md5 = c_strdup(result->vector[9]); c_strlist_destroy(result); return st; } +char *csync_statedb_get_uniqId( CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf ) { + char *ret = NULL; + c_strlist_t *result = NULL; + char *stmt = NULL; + + stmt = sqlite3_mprintf("SELECT md5 FROM metadata WHERE phash='%llu' AND modtime=%lu", jHash, buf->mtime); + + result = csync_statedb_query(ctx, stmt); + sqlite3_free(stmt); + if (result == NULL) { + return NULL; + } + + if (result->count == 1) { + /* phash, pathlen, path, inode, uid, gid, mode, modtime */ + ret = c_strdup( result->vector[0] ); + } + + c_strlist_destroy(result); + + return ret; +} + /* query the statedb, caller must free the memory */ c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) { int err = SQLITE_OK; @@ -475,6 +527,7 @@ c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) { sqlite3_stmt *stmt; const char *tail = NULL; c_strlist_t *result = NULL; + int row = 0; do { /* compile SQL program into a virtual machine, reattempteing if busy */ @@ -527,14 +580,20 @@ c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) { break; } - result = c_strlist_new(column_count); + row++; + if( result ) { + result = c_strlist_expand(result, row*column_count); + } else { + result = c_strlist_new(column_count); + } + if (result == NULL) { return NULL; } /* iterate over columns */ for (i = 0; i < column_count; i++) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_column_text: %s", (char *) sqlite3_column_text(stmt, i)); + // CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_column_text: %s", (char *) sqlite3_column_text(stmt, i)); if (c_strlist_add(result, (char *) sqlite3_column_text(stmt, i)) < 0) { c_strlist_destroy(result); return NULL; diff --git a/src/csync_statedb.h b/src/csync_statedb.h index 1ec6fe28e5..1ca303e311 100644 --- a/src/csync_statedb.h +++ b/src/csync_statedb.h @@ -61,6 +61,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash); csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode); +char *csync_statedb_get_uniqId( CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf ); /** * @brief A generic statedb query. * diff --git a/src/csync_update.c b/src/csync_update.c index 2b44932aef..997a19cdd4 100644 --- a/src/csync_update.c +++ b/src/csync_update.c @@ -103,37 +103,56 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, */ if (csync_get_statedb_exists(ctx)) { tmp = csync_statedb_get_stat_by_hash(ctx, h); - if (tmp && tmp->phash == h) { - /* we have an update! */ - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "time compare: %lu <-> %lu", - fs->mtime, tmp->modtime); - if (fs->mtime > tmp->modtime) { - st->instruction = CSYNC_INSTRUCTION_EVAL; - goto out; - } - st->instruction = CSYNC_INSTRUCTION_NONE; - } else { - /* check if it's a file and has been renamed */ - if (type == CSYNC_FTW_TYPE_FILE && ctx->current == LOCAL_REPLICA) { - tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode); - if (tmp && tmp->inode == fs->inode) { - /* inode found so the file has been renamed */ - st->instruction = CSYNC_INSTRUCTION_RENAME; - goto out; - } else { - /* file not found in statedb */ - st->instruction = CSYNC_INSTRUCTION_NEW; - goto out; +#if 0 + /* this code could possibly replace the one in csync_vio.c stat and would be more efficient */ + if(tmp) { + if( ctx->current == LOCAL_REPLICA ) { + if(fs->mtime == tmp->modtime && fs->size == tmp->size) { + /* filesystem modtime is still the same as the db mtime + * thus the md5 sum is still valid. */ + fs->md5 = c_strdup( tmp->md5 ); + } } - } - /* directory, remote and file not found in statedb */ - st->instruction = CSYNC_INSTRUCTION_NEW; + } +#endif + if(tmp && tmp->phash == h ) { /* there is an entry in the database */ + /* we have an update! */ + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "time compare: %lu <-> %lu, md5: %s <-> %s", + fs->mtime, tmp->modtime, fs->md5, tmp->md5); + if( !fs->md5) { + st->instruction = CSYNC_INSTRUCTION_EVAL; + goto out; + } + if( !c_streq(fs->md5, tmp->md5 )) { + // if (!fs->mtime > tmp->modtime) { + st->instruction = CSYNC_INSTRUCTION_EVAL; + goto out; + } + st->instruction = CSYNC_INSTRUCTION_NONE; + } else { + /* check if it's a file and has been renamed */ + if (type == CSYNC_FTW_TYPE_FILE && ctx->current == LOCAL_REPLICA) { + tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode); + if (tmp && tmp->inode == fs->inode) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "inodes: %ld <-> %ld", tmp->inode, fs->inode); + /* inode found so the file has been renamed */ + st->instruction = CSYNC_INSTRUCTION_RENAME; + goto out; + } else { + /* file not found in statedb */ + st->instruction = CSYNC_INSTRUCTION_NEW; + goto out; + } + } + /* directory, remote and file not found in statedb */ + st->instruction = CSYNC_INSTRUCTION_NEW; } } else { - st->instruction = CSYNC_INSTRUCTION_NEW; + st->instruction = CSYNC_INSTRUCTION_NEW; } out: + if( tmp) SAFE_FREE(tmp->md5); SAFE_FREE(tmp); st->inode = fs->inode; st->mode = fs->mode; @@ -143,7 +162,10 @@ out: st->gid = fs->gid; st->nlink = fs->nlink; st->type = type; - + st->md5 = NULL; + if( fs->md5 ) { + st->md5 = c_strdup(fs->md5); + } st->phash = h; st->pathlen = len; memcpy(st->path, (len ? path : ""), len + 1); @@ -204,6 +226,66 @@ int csync_walker(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs, return 0; } +/* check if the dirent entries for the directory can be read from db + * instead really calling readdir which is costly over net. + * For that, a single HEAD request is done on the directory to get its + * id. If the ID has not changed remotely, this subtree hasn't changed + * and can be read from db. + */ +static int _check_read_from_db(CSYNC *ctx, const char *uri) { + int len; + uint64_t h; + csync_vio_file_stat_t *fs = NULL; + char *md5_local = NULL; + const char *md5_remote = NULL; + const char *mpath; + c_rbnode_t *node = NULL; + int rc = 0; /* FIXME: Error handling! */ + + if( !c_streq( ctx->remote.uri, uri )) { + /* FIXME: The top uri can not be checked because there is no db entry for it */ + if( strlen(uri) < strlen(ctx->remote.uri)+1 ) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "check_read_from_db: uri is not a remote uri."); + /* FIXME: errno? */ + return -1; + } + mpath = uri + strlen(ctx->remote.uri) + 1; + fs = csync_vio_file_stat_new(); + if(fs == NULL) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "check_read_from_db: memory fault."); + errno = ENOMEM; + return -1; + } + len = strlen( mpath ); + h = c_jhash64((uint8_t *) mpath, len, 0); + + /* search in the local tree for the local file to get the mtime */ + node = c_rbtree_find(ctx->local.tree, &h); + if(node == NULL) { + /* no local file found. */ + } else { + csync_file_stat_t *other = NULL; + /* set the mtime which is needed in statedb_get_uniqid */ + other = (csync_file_stat_t *) node->data; + if( other ) + fs->mtime = other->modtime; + } + md5_local = csync_statedb_get_uniqId( ctx, h, fs ); + md5_remote = csync_vio_file_id(ctx, uri); + + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Compare directory ids for %s: %s -> %s", mpath, md5_local, md5_remote ); + + if( c_streq(md5_local, md5_remote) ) { + ctx->remote.read_from_db = 1; + } + SAFE_FREE(md5_local); + SAFE_FREE(md5_remote); + + csync_vio_file_stat_destroy(fs); + } + return rc; +} + /* File tree walker */ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, unsigned int depth) { @@ -213,13 +295,30 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, csync_vio_handle_t *dh = NULL; csync_vio_file_stat_t *dirent = NULL; csync_vio_file_stat_t *fs = NULL; + int read_from_db = 0; int rc = 0; + int res = 0; + + bool do_read_from_db = (ctx->current == REMOTE_REPLCIA && ctx->remote.read_from_db); if (uri[0] == '\0') { errno = ENOENT; goto error; } + /* If remote, compare the id with the local id. If equal, read all contents from + * the database. */ + read_from_db = ctx->remote.read_from_db; + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Incoming read_from_db-Flag for %s: %d", + uri, read_from_db ); + if( ctx->current == REMOTE_REPLCIA && !do_read_from_db ) { + _check_read_from_db(ctx, uri); + do_read_from_db = (ctx->current == REMOTE_REPLCIA && ctx->remote.read_from_db); + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Checking for read from db for %s: %d", + uri, ctx->remote.read_from_db ); + + } + if ((dh = csync_vio_opendir(ctx, uri)) == NULL) { /* permission denied */ if (errno == EACCES) { @@ -280,8 +379,16 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, continue; } - fs = csync_vio_file_stat_new(); - if (csync_vio_stat(ctx, filename, fs) == 0) { + /* == see if really stat has to be called. */ + if( do_read_from_db ) { + fs = dirent; + res = 0; + } else { + fs = csync_vio_file_stat_new(); + res = csync_vio_stat(ctx, filename, fs); + } + + if( res == 0) { switch (fs->type) { case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK: flag = CSYNC_FTW_FLAG_SLINK; @@ -305,11 +412,29 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, flag = CSYNC_FTW_FLAG_NSTAT; } + if( ctx->current == LOCAL_REPLICA ) { + char *md5 = NULL; + int len = strlen( path ); + uint64_t h = c_jhash64((uint8_t *) path, len, 0); + md5 = csync_statedb_get_uniqId( ctx, h, fs ); + if( md5 ) { + SAFE_FREE(fs->md5); + fs->md5 = md5; + fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; + } + + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID read from Database: %s -> %s", path, fs->md5 ? fs->md5 : "" ); + } + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "walk: %s", filename); /* Call walker function for each file */ rc = fn(ctx, filename, fs, flag); - csync_vio_file_stat_destroy(fs); + + if( ! do_read_from_db ) + csync_vio_file_stat_destroy(fs); + else + SAFE_FREE(fs->md5); if (rc < 0) { csync_vio_closedir(ctx, dh); @@ -328,6 +453,9 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn, dirent = NULL; } csync_vio_closedir(ctx, dh); + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Closing walk for %s with read_from_db %d", uri, read_from_db); + + ctx->remote.read_from_db = read_from_db; done: csync_vio_file_stat_destroy(dirent); diff --git a/src/csync_util.c b/src/csync_util.c index 0b68dbaaba..c4f49629b3 100644 --- a/src/csync_util.c +++ b/src/csync_util.c @@ -32,14 +32,6 @@ #include "csync_util.h" #include "vio/csync_vio.h" -#if defined(__APPLE__) -# define COMMON_DIGEST_FOR_OPENSSL -# include -# define SHA1 CC_SHA1 -#else -# include -#endif - #define CSYNC_LOG_CATEGORY_NAME "csync.util" #include "csync_log.h" @@ -113,6 +105,7 @@ void csync_memstat_check(void) { static int _merge_file_trees_visitor(void *obj, void *data) { csync_file_stat_t *fs = NULL; + csync_file_stat_t *tfs = NULL; csync_vio_file_stat_t *vst = NULL; CSYNC *ctx = NULL; @@ -181,25 +174,31 @@ static int _merge_file_trees_visitor(void *obj, void *data) { fs = c_rbtree_node_data(node); switch (ctx->current) { - case LOCAL_REPLICA: - if (asprintf(&uri, "%s/%s", ctx->local.uri, fs->path) < 0) { - rc = -1; - strerror_r(errno, errbuf, sizeof(errbuf)); - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s", - errbuf); - goto out; + case LOCAL_REPLICA: + /* If there is a destpath, this is a rename and the target must be used. */ + if( fs->destpath ) { + asprintf(&uri, "%s/%s", ctx->local.uri, fs->destpath); + SAFE_FREE(fs->destpath); + } else { + if (asprintf(&uri, "%s/%s", ctx->local.uri, fs->path) < 0) { + rc = -1; + strerror_r(errno, errbuf, sizeof(errbuf)); + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s", + errbuf); + goto out; + } } break; - case REMOTE_REPLCIA: + case REMOTE_REPLCIA: if (asprintf(&uri, "%s/%s", ctx->remote.uri, fs->path) < 0) { - rc = -1; - strerror_r(errno, errbuf, sizeof(errbuf)); - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s", - errbuf); - goto out; + rc = -1; + strerror_r(errno, errbuf, sizeof(errbuf)); + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s", + errbuf); + goto out; } break; - default: + default: break; } @@ -219,7 +218,24 @@ static int _merge_file_trees_visitor(void *obj, void *data) { fs->inode = vst->inode; fs->modtime = vst->mtime; - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: UPDATED", uri); + /* update with the id from the remote repo. This method always works on the local repo */ + node = c_rbtree_find(ctx->remote.tree, &fs->phash); + if (node == NULL) { + rc = -1; + CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Unable to find node"); + goto out; + } + + tfs = c_rbtree_node_data(node); + if( tfs && tfs->md5 ) { + if( fs->md5 ) SAFE_FREE(fs->md5); + fs->md5 = c_strdup( tfs->md5 ); + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "PRE UPDATED %s: %s", fs->path, fs->md5); + } else { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "md5 is empty in merger!"); + } + + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: UPDATED (%s)", uri, fs->md5); fs->instruction = CSYNC_INSTRUCTION_NONE; @@ -334,68 +350,4 @@ uint64_t csync_create_statedb_hash(CSYNC *ctx) { return hash; } -static char* digest_to_out(unsigned char* digest) -{ - char *out = c_malloc(33); - int n; - - if( !digest ) return NULL; - - for (n = 0; n < 16; ++n) { - snprintf(&(out[n*2]), 16*2, "%02x", (unsigned int) *(digest+n)); - } - - return out; -} - -#define BUF_SIZE 512 - -char* csync_file_md5(const char *filename) -{ - const char *tmpFileName; - int fd; - MD5_CTX c; - char buf[ BUF_SIZE+1 ]; - unsigned char digest[16]; - size_t size; - - tmpFileName = c_multibyte( filename ); - - if ( (fd = _topen( tmpFileName, O_RDONLY )) < 0) { - return NULL; - } else { - MD5_Init(&c); - while( (size=read(fd, buf, BUF_SIZE )) > 0) { - buf[size]='\0'; - MD5_Update(&c, buf, size); - } - close(fd); - MD5_Final(digest, &c); - } - - c_free_multibyte(tmpFileName); - return digest_to_out(digest); -} - -char* csync_buffer_md5(const char *str, int length) -{ - MD5_CTX c; - unsigned char digest[16]; - - MD5_Init(&c); - - while (length > 0) { - if (length > 512) { - MD5_Update(&c, str, 512); - } else { - MD5_Update(&c, str, length); - } - length -= 512; - str += 512; - } - - MD5_Final(digest, &c); - return digest_to_out(digest); -} - /* vim: set ts=8 sw=2 et cindent: */ diff --git a/src/csync_util.h b/src/csync_util.h index 96727b4e6e..e7231b85db 100644 --- a/src/csync_util.h +++ b/src/csync_util.h @@ -36,13 +36,5 @@ int csync_unix_extensions(CSYNC *ctx); /* Normalize the uri to / */ uint64_t csync_create_statedb_hash(CSYNC *ctx); -/* Calculate the md5 sum for a file given by filename. - * Caller has to free the memory. */ -char* csync_file_md5(const char *filename); - -/* Create an md5 sum from a data pointer with a given length. - * Caller has to free the memory */ -char* csync_buffer_md5(const char *str, int length); - #endif /* _CSYNC_UTIL_H */ /* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */ diff --git a/src/vio/csync_vio.c b/src/vio/csync_vio.c index 48d0428b72..cb095d0839 100644 --- a/src/vio/csync_vio.c +++ b/src/vio/csync_vio.c @@ -29,9 +29,12 @@ #include /* dlopen(), dlclose(), dlsym() ... */ #include "csync_private.h" +#include "csync_dbtree.h" #include "vio/csync_vio.h" #include "vio/csync_vio_handle_private.h" #include "vio/csync_vio_local.h" +#include "csync_statedb.h" +#include "std/c_jhash.h" #define CSYNC_LOG_CATEGORY_NAME "csync.vio.main" @@ -193,6 +196,11 @@ int csync_vio_init(CSYNC *ctx, const char *module, const char *args) { return -1; } + if (! VIO_METHOD_HAS_FUNC(m, get_file_id)) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "module %s has no get_file_id fn", module); + } + + ctx->module.method = m; return 0; @@ -353,7 +361,12 @@ csync_vio_handle_t *csync_vio_opendir(CSYNC *ctx, const char *name) { switch(ctx->replica) { case REMOTE_REPLCIA: - mh = ctx->module.method->opendir(name); + if(ctx->remote.read_from_db) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Reading directory %s from database", name); + mh = csync_dbtree_opendir(ctx, name); + } else { + mh = ctx->module.method->opendir(name); + } break; case LOCAL_REPLICA: mh = csync_vio_local_opendir(name); @@ -380,7 +393,11 @@ int csync_vio_closedir(CSYNC *ctx, csync_vio_handle_t *dhandle) { switch(ctx->replica) { case REMOTE_REPLCIA: - rc = ctx->module.method->closedir(dhandle->method_handle); + if(ctx->remote.read_from_db) { + rc = csync_dbtree_closedir(ctx, dhandle->method_handle); + } else { + rc = ctx->module.method->closedir(dhandle->method_handle); + } break; case LOCAL_REPLICA: rc = csync_vio_local_closedir(dhandle->method_handle); @@ -400,7 +417,11 @@ csync_vio_file_stat_t *csync_vio_readdir(CSYNC *ctx, csync_vio_handle_t *dhandle switch(ctx->replica) { case REMOTE_REPLCIA: - fs = ctx->module.method->readdir(dhandle->method_handle); + if(ctx->remote.read_from_db) { + fs = csync_dbtree_readdir(ctx, dhandle->method_handle); + } else { + fs = ctx->module.method->readdir(dhandle->method_handle); + } break; case LOCAL_REPLICA: fs = csync_vio_local_readdir(dhandle->method_handle); @@ -508,6 +529,17 @@ int csync_vio_rmdir(CSYNC *ctx, const char *uri) { return rc; } +const char *csync_vio_file_id(CSYNC *ctx, const char *path) +{ + const char *re = NULL; + /* We always use the remote method here. */ + if(ctx->module.method && + VIO_METHOD_HAS_FUNC(ctx->module.method, get_file_id)) { + re = ctx->module.method->get_file_id(path); + } + return re; +} + int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) { int rc = -1; @@ -520,7 +552,6 @@ int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) { #ifdef _WIN32 CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Win32: STAT-inode for %s: %llu", uri, buf->inode ); #endif - break; default: break; diff --git a/src/vio/csync_vio.h b/src/vio/csync_vio.h index 9db6a1ca1d..715ed626e1 100644 --- a/src/vio/csync_vio.h +++ b/src/vio/csync_vio.h @@ -48,6 +48,7 @@ int csync_vio_mkdir(CSYNC *ctx, const char *uri, mode_t mode); int csync_vio_mkdirs(CSYNC *ctx, const char *uri, mode_t mode); int csync_vio_rmdir(CSYNC *ctx, const char *uri); +const char *csync_vio_file_id(CSYNC *ctx, const char *path); int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf); int csync_vio_rename(CSYNC *ctx, const char *olduri, const char *newuri); int csync_vio_unlink(CSYNC *ctx, const char *uri); diff --git a/src/vio/csync_vio_file_stat.c b/src/vio/csync_vio_file_stat.c index be8d8f4afe..352b8e83ce 100644 --- a/src/vio/csync_vio_file_stat.c +++ b/src/vio/csync_vio_file_stat.c @@ -30,7 +30,7 @@ csync_vio_file_stat_t *csync_vio_file_stat_new(void) { if (file_stat == NULL) { return NULL; } - + file_stat->md5 = NULL; return file_stat; } @@ -40,13 +40,15 @@ void csync_vio_file_stat_destroy(csync_vio_file_stat_t *file_stat) { return; } - if (file_stat->fields == CSYNC_VIO_FILE_STAT_FIELDS_SYMLINK_NAME) { + if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_SYMLINK_NAME) { SAFE_FREE(file_stat->u.symlink_name); } - if (file_stat->fields == CSYNC_VIO_FILE_STAT_FIELDS_CHECKSUM) { + if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_CHECKSUM) { SAFE_FREE(file_stat->u.checksum); } - + if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_MD5) { + SAFE_FREE(file_stat->md5); + } SAFE_FREE(file_stat->name); SAFE_FREE(file_stat); } diff --git a/src/vio/csync_vio_file_stat.h b/src/vio/csync_vio_file_stat.h index 09e59280ab..93a7f7084c 100644 --- a/src/vio/csync_vio_file_stat.h +++ b/src/vio/csync_vio_file_stat.h @@ -72,6 +72,7 @@ enum csync_vio_file_stat_fields_e { CSYNC_VIO_FILE_STAT_FIELDS_ACL = 1 << 14, CSYNC_VIO_FILE_STAT_FIELDS_UID = 1 << 15, CSYNC_VIO_FILE_STAT_FIELDS_GID = 1 << 16, + CSYNC_VIO_FILE_STAT_FIELDS_MD5 = 1 << 17, }; @@ -83,6 +84,7 @@ struct csync_vio_file_stat_s { void *acl; char *name; + char *md5; uid_t uid; gid_t gid; diff --git a/src/vio/csync_vio_local.c b/src/vio/csync_vio_local.c index 8835f03ee7..1b700e1864 100644 --- a/src/vio/csync_vio_local.c +++ b/src/vio/csync_vio_local.c @@ -35,6 +35,8 @@ #include "c_private.h" #include "c_lib.h" #include "c_string.h" +#include "csync_util.h" +#include "csync_log.h" #include "vio/csync_vio_local.h" @@ -387,6 +389,11 @@ int csync_vio_local_stat(const char *uri, csync_vio_file_stat_t *buf) { buf->ctime = sb.st_ctime; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_CTIME; + /* buf->md5 = csync_file_md5( uri ); */ + /* md5 is queried one level higher because we do not have ctx here. */ + + buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5; + c_free_multibyte(wuri); return 0; } diff --git a/src/vio/csync_vio_method.h b/src/vio/csync_vio_method.h index 814504f98d..ba1d5d84f0 100644 --- a/src/vio/csync_vio_method.h +++ b/src/vio/csync_vio_method.h @@ -52,7 +52,7 @@ typedef csync_vio_method_t *(*csync_vio_method_init_fn)(const char *method_name, typedef void (*csync_vio_method_finish_fn)(csync_vio_method_t *method); typedef csync_vio_capabilities_t *(*csync_method_get_capabilities_fn)(void); - +typedef const char* (*csync_method_get_file_id_fn)(const char* path); typedef csync_vio_method_handle_t *(*csync_method_open_fn)(const char *durl, int flags, mode_t mode); typedef csync_vio_method_handle_t *(*csync_method_creat_fn)(const char *durl, mode_t mode); typedef int (*csync_method_close_fn)(csync_vio_method_handle_t *fhandle); @@ -79,6 +79,7 @@ typedef int (*csync_method_utimes_fn)(const char *uri, const struct timeval time struct csync_vio_method_s { size_t method_table_size; /* Used for versioning */ csync_method_get_capabilities_fn get_capabilities; + csync_method_get_file_id_fn get_file_id; csync_method_open_fn open; csync_method_creat_fn creat; csync_method_close_fn close; diff --git a/tests/csync_tests/check_csync_exclude.c b/tests/csync_tests/check_csync_exclude.c index e850321cb2..77862ae273 100644 --- a/tests/csync_tests/check_csync_exclude.c +++ b/tests/csync_tests/check_csync_exclude.c @@ -46,6 +46,12 @@ START_TEST (check_csync_excluded) fail_unless(csync_excluded(csync, ".kde/share/config/kwin.eventsrc") == 0, NULL); fail_unless(csync_excluded(csync, ".kde4/cache-maximegalon/cache1.txt") == 1, NULL); fail_unless(csync_excluded(csync, ".mozilla/plugins") == 1, NULL); + + /* test for finding patterns in subdirs. .beagle is defined as a pattern and has to + * be found in top dir as well as in directories underneath. */ + fail_unless(csync_excluded(csync, ".beagle") == 1, NULL); + fail_unless(csync_excluded(csync, "foo/.beagle") == 1, NULL); + fail_unless(csync_excluded(csync, "bar/foo/.beagle") == 1, NULL); } END_TEST diff --git a/tests/csync_tests/check_csync_statedb_query.c b/tests/csync_tests/check_csync_statedb_query.c index c312766c45..0ce37a330e 100644 --- a/tests/csync_tests/check_csync_statedb_query.c +++ b/tests/csync_tests/check_csync_statedb_query.c @@ -36,8 +36,8 @@ static void setup_db(void) { fail_unless(csync_statedb_create_tables(csync) == 0, "Setup failed"); stmt = sqlite3_mprintf("INSERT INTO metadata" - "(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES" - "(%lu, %d, '%q', %d, %d, %d, %d, %lu);", + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES" + "(%lu, %d, '%q', %d, %d, %d, %d, %lu, %d, %lu);", 42, 42, "It's a rainy day", @@ -45,7 +45,9 @@ static void setup_db(void) { 42, 42, 42, - 42); + 42, + 2, + 43); fail_if(csync_statedb_insert(csync, stmt) < 0, NULL); sqlite3_free(stmt); diff --git a/tests/csync_tests/check_csync_treewalk.c b/tests/csync_tests/check_csync_treewalk.c index 2facd9f58a..5f985bdfc9 100644 --- a/tests/csync_tests/check_csync_treewalk.c +++ b/tests/csync_tests/check_csync_treewalk.c @@ -100,7 +100,7 @@ START_TEST (check_csync_treewalk_local_with_filter) fail_if(file_count != 2, "Local File count filtered (EVAL) not correct: %d", file_count); file_count = 0; fail_if(csync_walk_local_tree(csync, &visitor, CSYNC_INSTRUCTION_EVAL | CSYNC_INSTRUCTION_REMOVE) < 0, "Local walk"); - fail_if(file_count != 2, "Local File count filtered (EVAL) not correct: %d", file_count); + fail_if(file_count != 2, "Local File count filtered (EVAL|REMOVE) not correct: %d", file_count); file_count = 0; fail_if(csync_walk_local_tree(csync, &visitor, CSYNC_INSTRUCTION_RENAME) < 0, "Local walk"); fail_if(file_count != 0, "Local File count filtered (RENAME) not correct"); diff --git a/tests/csync_tests/check_csync_update.c b/tests/csync_tests/check_csync_update.c index 7a6006aa9e..fc7ba5f017 100644 --- a/tests/csync_tests/check_csync_update.c +++ b/tests/csync_tests/check_csync_update.c @@ -153,7 +153,7 @@ START_TEST (check_csync_detect_update_db_none) /* the instruction should be set to new */ st = c_rbtree_node_data(csync->local.tree->root); - fail_unless(st->instruction == CSYNC_INSTRUCTION_NONE, "instruction is %s", csync_instruction_str(st->instruction)); + fail_unless(st->instruction == CSYNC_INSTRUCTION_EVAL, "instruction is %s", csync_instruction_str(st->instruction)); /* set the instruction to UPDATED that it gets written to the statedb */ st->instruction = CSYNC_INSTRUCTION_UPDATED; diff --git a/tests/ownCloud/HTTP/DAV.pm b/tests/ownCloud/HTTP/DAV.pm new file mode 100644 index 0000000000..cdc95ce196 --- /dev/null +++ b/tests/ownCloud/HTTP/DAV.pm @@ -0,0 +1,2075 @@ +# +# Perl WebDAV client library +# + +package HTTP::DAV; + +use LWP; +use XML::DOM; +use Time::Local; +use HTTP::DAV::Lock; +use HTTP::DAV::ResourceList; +use HTTP::DAV::Resource; +use HTTP::DAV::Comms; +use URI::file; +use URI::Escape; +use FileHandle; +use File::Glob; + +#use Carp (cluck); +use Cwd (); # Can't import all of it, cwd clashes with our namespace. + +# Globals +$VERSION = '0.44'; +$VERSION_DATE = '2011/06/19'; + +# Set this up to 3 +$DEBUG = 0; + +use strict; +use vars qw($VERSION $VERSION_DATE $DEBUG); + +sub new { + my $class = shift; + my $self = bless {}, ref($class) || $class; + $self->_init(@_); + return $self; +} + +########################################################################### +sub clone { + my $self = @_; + my $class = ref($self); + my %clone = %{$self}; + bless {%clone}, $class; +} + +########################################################################### +{ + + sub _init { + my ( $self, @p ) = @_; + my ( $uri, $headers, $useragent ) + = HTTP::DAV::Utils::rearrange( [ 'URI', 'HEADERS', 'USERAGENT' ], + @p ); + + $self->{_lockedresourcelist} = HTTP::DAV::ResourceList->new(); + $self->{_comms} = HTTP::DAV::Comms->new( + -useragent => $useragent, + -headers => $headers + ); + if ($uri) { + $self->set_workingresource( $self->new_resource( -uri => $uri ) ); + } + + return $self; + } +} + +sub DebugLevel { + shift if ref( $_[0] ) =~ /HTTP/; + my $level = shift; + $level = 256 if !defined $level || $level eq ""; + + $DEBUG = $level; +} + +###################################################################### +# new_resource acts as a resource factory. +# It will create a new one for you each time you ask. +# Sometimes, if it holds state information about this +# URL, it may return an old populated object. +sub new_resource { + my ($self) = shift; + + #### + # This is the order of the arguments unless used as + # named parameters + my ($uri) = HTTP::DAV::Utils::rearrange( ['URI'], @_ ); + $uri = HTTP::DAV::Utils::make_uri($uri); + + #cluck "new_resource: now $uri\n"; + + my $resource = $self->{_lockedresourcelist}->get_member($uri); + if ($resource) { + print + "new_resource: For $uri, returning existing resource $resource\n" + if $HTTP::DAV::DEBUG > 2; + + # Just reset the url to honour trailing slash status. + $resource->set_uri($uri); + return $resource; + } + else { + print "new_resource: For $uri, creating new resource\n" + if $HTTP::DAV::DEBUG > 2; + return HTTP::DAV::Resource->new( + -Comms => $self->{_comms}, + -LockedResourceList => $self->{_lockedresourcelist}, + -uri => $uri, + -Client => $self + ); + } +} + +########################################################################### +# ACCESSOR METHODS + +# GET +sub get_user_agent { $_[0]->{_comms}->get_user_agent(); } +sub get_last_request { $_[0]->{_comms}->get_last_request(); } +sub get_last_response { $_[0]->{_comms}->get_last_response(); } +sub get_workingresource { $_[0]->{_workingresource} } + +sub get_workingurl { + $_[0]->{_workingresource}->get_uri() + if defined $_[0]->{_workingresource}; +} +sub get_lockedresourcelist { $_[0]->{_lockedresourcelist} } + +# SET +sub set_workingresource { $_[0]->{_workingresource} = $_[1]; } +sub credentials { shift->{_comms}->credentials(@_); } + +###################################################################### +# Error handling + +## Error conditions +my %err = ( + 'ERR_WRONG_ARGS' => 'Wrong number of arguments supplied.', + 'ERR_UNAUTHORIZED' => 'Unauthorized. ', + 'ERR_NULL_RESOURCE' => 'Not connected. Do an open first. ', + 'ERR_RESP_FAIL' => 'Server response: ', + 'ERR_501' => 'Server response: ', + 'ERR_405' => 'Server response: ', + 'ERR_GENERIC' => '', +); + +sub err { + my ( $self, $error, $mesg, $url ) = @_; + + my $err_msg; + $err_msg = ""; + $err_msg .= $err{$error} if defined $err{$error}; + $err_msg .= $mesg if defined $mesg; + $err_msg .= "ERROR" unless defined $err_msg; + + $self->{_message} = $err_msg; + my $callback = $self->{_callback}; + &$callback( 0, $err_msg, $url ) if $callback; + + if ( $self->{_multi_op} ) { + push( @{ $self->{_errors} }, $err_msg ); + } + $self->{_status} = 0; + + return 0; +} + +sub ok { + my ($self, $mesg, $url, $so_far, $length) = @_; + + $self->{_message} = $mesg; + + my $callback = $self->{_callback}; + &$callback(1, $mesg, $url, $so_far, $length) if $callback; + + if ($self->{_multi_op}) { + $self->{_status} = 1 unless $self->{_status} == 0; + } + else { + $self->{_status} = 1; + } + return 1; +} + +sub _start_multi_op { + my ($self, $mesg, $callback) = @_; + $self->{_multi_mesg} = $mesg || ""; + $self->{_status} = 1; + $self->{_errors} = []; + $self->{_multi_op} = 1; + $self->{_callback} = $callback if defined $callback; +} + +sub _end_multi_op { + my ($self) = @_; + $self->{_multi_op} = 0; + $self->{_callback} = undef; + my $message = $self->{_multi_mesg} . " "; + $message .= ( $self->{_status} ) ? "succeeded" : "failed"; + $self->{_message} = $message; + $self->{_multi_mesg} = undef; +} + +sub message { + my ($self) = @_; + return $self->{_message} || ""; +} + +sub errors { + my ($self) = @_; + my $err_ref = $self->{_errors} || []; + return @{ $err_ref }; +} + +sub is_success { + my ($self) = @_; + return $self->{_status}; +} + +###################################################################### +# Operations + +# CWD +sub cwd { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + return $self->err('ERR_WRONG_ARGS') if ( !defined $url || $url eq "" ); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + $url = HTTP::DAV::Utils::make_trail_slash($url); + my $new_uri = $self->get_absolute_uri($url); + ($new_uri) = $self->get_globs($new_uri); + + return 0 unless ($new_uri); + + print "cwd: Changing to $new_uri\n" if $DEBUG; + return $self->open($new_uri); +} + +# DELETE +sub delete { + my ( $self, @p ) = @_; + my ( $url, $callback ) + = HTTP::DAV::Utils::rearrange( [ 'URL', 'CALLBACK' ], @p ); + + return $self->err('ERR_WRONG_ARGS') if ( !defined $url || $url eq "" ); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $new_url = $self->get_absolute_uri($url); + my @urls = $self->get_globs($new_url); + + $self->_start_multi_op( "delete $url", $callback ) if @urls > 1; + + foreach my $u (@urls) { + my $resource = $self->new_resource( -uri => $u ); + + my $resp = $resource->delete(); + + if ( $resp->is_success ) { + $self->ok( "deleted $u successfully", $u ); + } + else { + $self->err( 'ERR_RESP_FAIL', $resp->message(), $u ); + } + } + + $self->_end_multi_op() if @urls > 1; + + return $self->is_success; +} + +# GET +# Handles globs by doing multiple recursive gets +# GET dir* produces +# _get dir1, to_local +# _get dir2, to_local +# _get dir3, to_local +sub get { + my ( $self, @p ) = @_; + my ( $url, $to, $callback, $chunk ) + = HTTP::DAV::Utils::rearrange( [ 'URL', 'TO', 'CALLBACK', 'CHUNK' ], + @p ); + + return $self->err('ERR_WRONG_ARGS') if ( !defined $url || $url eq "" ); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + $self->_start_multi_op( "get $url", $callback ); + + my $new_url = $self->get_absolute_uri($url); + my (@urls) = $self->get_globs($new_url); + + return 0 unless ( $#urls > -1 ); + + ############ + # HANDLE -TO + # + $to ||= ''; + if ( $to eq '.' ) { + $to = Cwd::getcwd(); + } + + # If the TO argument is a file handle or a scalar + # then check that we only got one glob. If we got multiple + # globs, then we can't keep going because we can't write multiple files + # to one FileHandle. + if ( $#urls > 0 ) { + if ( ref($to) =~ /SCALAR/ ) { + return $self->err( 'ERR_WRONG_ARGS', + "Can't retrieve multiple files to a single scalar\n" ); + } + elsif ( ref($to) =~ /GLOB/ ) { + return $self->err( 'ERR_WRONG_ARGS', + "Can't retrieve multiple files to a single filehandle\n" ); + } + } + + # If it's a dir, remove last '/' from destination. + # Later we need to concatenate the destination filename. + if ( defined $to && $to ne '' && -d $to ) { + $to =~ s{/$}{}; + } + + # Foreach file... do the get. + foreach my $u (@urls) { + my ( $left, $leafname ) = HTTP::DAV::Utils::split_leaf($u); + + # Handle SCALARREF and GLOB cases + my $dest_file = $to; + + # Directories + if ( -d $to ) { + $dest_file = "$to/$leafname"; + + # Multiple targets + } + elsif ( !defined $to || $to eq "" ) { + $dest_file = $leafname; + } + + warn "get: $u -> $dest_file\n" if $DEBUG; + + # Setup the resource based on the passed url and do a propfind. + my $resource = $self->new_resource( -uri => $u ); + my $resp = $resource->propfind( -depth => 1 ); + + if ( $resp->is_error ) { + return $self->err( 'ERR_RESP_FAIL', $resp->message(), $u ); + } + + $self->_get( $resource, $dest_file, $callback, $chunk ); + } + + $self->_end_multi_op(); + return $self->is_success; +} + +# Note: is is expected that $resource has had +# a propfind depth 1 performed on it. +# +sub _get { + my ( $self, @p ) = @_; + my ( $resource, $local_name, $callback, $chunk ) + = HTTP::DAV::Utils::rearrange( + [ 'RESOURCE', 'TO', 'CALLBACK', 'CHUNK' ], @p ); + + my $url = $resource->get_uri(); + + # GET A DIRECTORY + if ( $resource->is_collection ) { + + # If the TO argument is a file handle, a scalar or empty + # then we + # can't keep going because we can't write multiple files + # to one FileHandle, scalar, etc. + if ( ref($local_name) =~ /SCALAR/ ) { + return $self->err( 'ERR_WRONG_ARGS', + "Can't retrieve a collection to a scalar\n", $url ); + } + elsif ( ref($local_name) =~ /GLOB/ ) { + return $self->err( 'ERR_WRONG_ARGS', + "Can't retrieve a collection to a filehandle\n", $url ); + } + elsif ( $local_name eq "" ) { + return $self->err( + 'ERR_GENERIC', + "Can't retrieve a collection without a target directory (-to).", + $url + ); + } + + # Try and make the directory locally + print "MKDIR $local_name (before escape)\n" if $DEBUG > 2; + + $local_name = URI::Escape::uri_unescape($local_name); + if ( !mkdir $local_name ) { + return $self->err( 'ERR_GENERIC', + "mkdir local:$local_name failed: $!" ); + } + + $self->ok("mkdir $local_name"); + + # This is the degenerate case for an empty dir. + print "Made directory $local_name\n" if $DEBUG > 2; + + my $resource_list = $resource->get_resourcelist(); + if ($resource_list) { + + # FOREACH FILE IN COLLECTION, GET IT. + foreach my $progeny_r ( $resource_list->get_resources() ) { + + my $progeny_url = $progeny_r->get_uri(); + print "Found progeny:$progeny_url\n" if $DEBUG > 2; + my $progeny_local_filename + = HTTP::DAV::Utils::get_leafname($progeny_url); + $progeny_local_filename + = URI::Escape::uri_unescape($progeny_local_filename); + + $progeny_local_filename + = URI::file->new($progeny_local_filename) + ->abs("$local_name/"); + + if ( $progeny_r->is_collection() ) { + $progeny_r->propfind( -depth => 1 ); + } + $self->_get( $progeny_r, $progeny_local_filename, $callback, + $chunk ); + + # } else { + # $self->_do_get_tofile($progeny_r,$progeny_local_filename); + # } + } + } + } + + # GET A FILE + else { + my $response; + my $name_ref = ref $local_name; + + if ( $callback || $name_ref =~ /SCALAR/ || $name_ref =~ /GLOB/ ) { + $self->{_so_far} = 0; + + my $fh; + my $put_to_scalar = 0; + + if ( $name_ref =~ /GLOB/ ) { + $fh = $local_name; + } + + elsif ( $name_ref =~ /SCALAR/ ) { + $put_to_scalar = 1; + $$local_name = ""; + } + + else { + $fh = FileHandle->new; + $local_name = URI::Escape::uri_unescape($local_name); + if (! $fh->open(">$local_name") ) { + return $self->err( + 'ERR_GENERIC', + "open \">$local_name\" failed: $!", + $url + ); + } + + # RT #29788, avoid file corruptions on Win32 + binmode $fh; + } + + $self->{_fh} = $fh; + + $response = $resource->get( + -chunk => $chunk, + -progress_callback => + + sub { + my ( $data, $response, $protocol ) = @_; + + $self->{_so_far} += length($data); + + my $fh = $self->{_fh}; + print $fh $data if defined $fh; + + $$local_name .= $data if ($put_to_scalar); + + my $user_callback = $self->{_callback}; + &$user_callback( -1, "transfer in progress", + $url, $self->{_so_far}, $response->content_length(), + $data ) + if defined $user_callback; + + } + + ); # end get( ... ); + + # Close the filehandle if it was set. + if ( defined $self->{_fh} ) { + $self->{_fh}->close(); + delete $self->{_fh}; + } + } + else { + $local_name = URI::Escape::uri_unescape($local_name); + $response = $resource->get( -save_to => $local_name ); + } + + # Handle response + if ( $response->is_error ) { + return $self->err( 'ERR_GENERIC', + "get $url failed: " . $response->message, $url ); + } + else { + return $self->ok( "get $url", $url, $self->{_so_far}, + $response->content_length() ); + } + + } + + return 1; +} + +# LOCK +sub lock { + my ( $self, @p ) = @_; + my ( $url, $owner, $depth, $timeout, $scope, $type, @other ) + = HTTP::DAV::Utils::rearrange( + [ 'URL', 'OWNER', 'DEPTH', 'TIMEOUT', 'SCOPE', 'TYPE' ], @p ); + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + $url = $resource->get_uri; + } + + # Make the lock + my $resp = $resource->lock( + -owner => $owner, + -depth => $depth, + -timeout => $timeout, + -scope => $scope, + -type => $type + ); + + if ( $resp->is_success() ) { + return $self->ok( "lock $url succeeded", $url ); + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message, $url ); + } +} + +# UNLOCK +sub unlock { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + $url = $resource->get_uri; + } + + # Make the lock + my $resp = $resource->unlock(); + if ( $resp->is_success ) { + return $self->ok( "unlock $url succeeded", $url ); + } + else { + + # The Resource.pm::lock routine has a hack + # where if it doesn't know the locktoken, it will + # just return an empty response with message "Client Error". + # Make a custom message for this case. + my $msg = $resp->message; + if ( $msg =~ /Client error/i ) { + $msg = "No locks found. Try steal"; + return $self->err( 'ERR_GENERIC', $msg, $url ); + } + else { + return $self->err( 'ERR_RESP_FAIL', $msg, $url ); + } + } +} + +sub steal { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + } + + # Go the steal + my $resp = $resource->forcefully_unlock_all(); + if ( $resp->is_success() ) { + return $self->ok( "steal succeeded", $url ); + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message(), $url ); + } +} + +# MKCOL +sub mkcol { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + return $self->err('ERR_WRONG_ARGS') if ( !defined $url || $url eq "" ); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + $url = HTTP::DAV::Utils::make_trail_slash($url); + my $new_url = $self->get_absolute_uri($url); + my $resource = $self->new_resource( -uri => $new_url ); + + # Make the lock + my $resp = $resource->mkcol(); + if ( $resp->is_success() ) { + return $self->ok( "mkcol $new_url", $new_url ); + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message(), $new_url ); + } +} + +# OPTIONS +sub options { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + #return $self->err('ERR_WRONG_ARGS') if (!defined $url || $url eq ""); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + $url = $resource->get_uri; + } + + # Make the call + my $resp = $resource->options(); + if ( $resp->is_success() ) { + $self->ok( "options $url succeeded", $url ); + return $resource->get_options(); + } + else { + $self->err( 'ERR_RESP_FAIL', $resp->message(), $url ); + return undef; + } +} + +# MOVE +sub move { return shift->_move_copy( "move", @_ ); } +sub copy { return shift->_move_copy( "copy", @_ ); } + +sub _move_copy { + my ( $self, $method, @p ) = @_; + my ( $url, $dest_url, $overwrite, $depth, $text, @other ) + = HTTP::DAV::Utils::rearrange( + [ 'URL', 'DEST', 'OVERWRITE', 'DEPTH', 'TEXT' ], @p ); + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + if (!( defined $url && $url ne "" && defined $dest_url && $dest_url ne "" + ) + ) + { + return $self->err( 'ERR_WRONG_ARGS', + "Must supply a source and destination url" ); + } + + $url = $self->get_absolute_uri($url); + $dest_url = $self->get_absolute_uri($dest_url); + my $resource = $self->new_resource( -uri => $url ); + my $dest_resource = $self->new_resource( -uri => $dest_url ); + + my $resp = $dest_resource->propfind( -depth => 1 ); + if ( $resp->is_success && $dest_resource->is_collection ) { + my $leafname = HTTP::DAV::Utils::get_leafname($url); + $dest_url = "$dest_url/$leafname"; + $dest_resource = $self->new_resource( -uri => $dest_url ); + } + + # Make the lock + $resp = $resource->$method( + -dest => $dest_resource, + -overwrite => $overwrite, + -depth => $depth, + -text => $text, + ); + + if ( $resp->is_success() ) { + return $self->ok( "$method $url to $dest_url succeeded", $url ); + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message, $url ); + } +} + +# OPEN +# Must be a collection resource +# $dav->open( -url => http://localhost/test/ ); +# $dav->open( localhost/test/ ); +# $dav->open( -url => localhost:81 ); +# $dav->open( localhost ); +sub open { + my ( $self, @p ) = @_; + my ($url) = HTTP::DAV::Utils::rearrange( ['URL'], @p ); + + my $resource; + if ( defined $url && $url ne "" ) { + $url = HTTP::DAV::Utils::make_trail_slash($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + $url = $resource->get_uri() if ($resource); + return $self->err('ERR_WRONG_ARGS') + if ( !defined $url || $url eq "" ); + } + + my $response = $resource->propfind( -depth => 0 ); + #print $response->as_string; + # print $resource->as_string; + + my $result = $self->what_happened($url, $resource, $response); + if ($result->{success} == 0) { + return $self->err($result->{error_type}, $result->{error_msg}, $url); + } + + # If it is a collection but the URI doesn't end in a trailing slash. + # Then we need to reopen with the / + elsif ($resource->is_collection + && $url !~ m#/\s*$# ) + { + my $newurl = $url . "/"; + print "Redirecting to $newurl\n" if $DEBUG > 1; + return $self->open($newurl); + } + + # If it is not a collection then we + # can't open it. + # elsif ( !$resource->is_collection ) { + # return $self->err( 'ERR_GENERIC', + # "Operation failed. You can only open a collection (directory)", + # $url ); + # } + else { + $self->set_workingresource($resource); + return $self->ok( "Connected to $url", $url ); + } + + return $self->err( 'ERR_GENERIC', $url ); +} + +# Performs a propfind and then returns the populated +# resource. The resource will have a resourcelist if +# it is a collection. +sub propfind { + my ( $self, @p ) = @_; + my ( $url, $depth ) = HTTP::DAV::Utils::rearrange( [ 'URL', 'DEPTH' ], @p ); + + # depth = 1 is the default + if (! defined $depth) { + $depth = 1; + } + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + } + + # Make the call + my $resp = $resource->propfind( -depth => $depth ); + if ( $resp->is_success() ) { + $resource->build_ls($resource); + $self->ok( "propfind " . $resource->get_uri() . " succeeded", $url ); + return $resource; + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message(), $url ); + } +} + +# Set a property on the resource +sub set_prop { + my ( $self, @p ) = @_; + my ( $url, $namespace, $propname, $propvalue, $nsabbr ) + = HTTP::DAV::Utils::rearrange( + [ 'URL', 'NAMESPACE', 'PROPNAME', 'PROPVALUE', 'NSABBR' ], @p ); + $self->proppatch( + -url => $url, + -namespace => $namespace, + -propname => $propname, + -propvalue => $propvalue, + -action => "set", + -nsabbr => $nsabbr, + ); +} + +# Unsets a property on the resource +sub unset_prop { + my ( $self, @p ) = @_; + my ( $url, $namespace, $propname, $nsabbr ) + = HTTP::DAV::Utils::rearrange( + [ 'URL', 'NAMESPACE', 'PROPNAME', 'NSABBR' ], @p ); + $self->proppatch( + -url => $url, + -namespace => $namespace, + -propname => $propname, + -action => "remove", + -nsabbr => $nsabbr, + ); +} + +# Performs a proppatch on the resource +sub proppatch { + my ( $self, @p ) = @_; + my ( $url, $namespace, $propname, $propvalue, $action, $nsabbr ) + = HTTP::DAV::Utils::rearrange( + [ 'URL', 'NAMESPACE', 'PROPNAME', 'PROPVALUE', 'ACTION', 'NSABBR' ], + @p ); + + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + my $resource; + if ($url) { + $url = $self->get_absolute_uri($url); + $resource = $self->new_resource( -uri => $url ); + } + else { + $resource = $self->get_workingresource(); + } + + # Make the call + my $resp = $resource->proppatch( + -namespace => $namespace, + -propname => $propname, + -propvalue => $propvalue, + -action => $action, + -nsabbr => $nsabbr + ); + + if ( $resp->is_success() ) { + $resource->build_ls($resource); + $self->ok( "proppatch " . $resource->get_uri() . " succeeded", $url ); + return $resource; + } + else { + return $self->err( 'ERR_RESP_FAIL', $resp->message(), $url ); + } +} + +###################################################################### +sub put { + my ( $self, @p ) = @_; + my ( $local, $url, $callback, $custom_headers ) + = HTTP::DAV::Utils::rearrange( [ 'LOCAL', 'URL', 'CALLBACK', 'HEADERS' ], @p ); + + if ( ref($local) eq "SCALAR" ) { + $self->_start_multi_op( 'put ' . ${$local}, $callback ); + $self->_put(@p); + } + else { + $self->_start_multi_op( 'put ' . $local, $callback ); + $local =~ s/\ /\\ /g; + my @globs = glob("$local"); + foreach my $file (@globs) { + print "Starting put of $file\n" if $HTTP::DAV::DEBUG > 1; + $self->_put( + -local => $file, + -url => $url, + -callback => $callback, + -headers => $custom_headers, + ); + } + } + $self->_end_multi_op(); + return $self->is_success; +} + +sub _put { + my ( $self, @p ) = @_; + my ( $local, $url, $custom_headers ) + = HTTP::DAV::Utils::rearrange( [ 'LOCAL', 'URL', 'HEADERS' ], @p ); + + return $self->err('ERR_WRONG_ARGS') + if ( !defined $local || $local eq "" ); + return $self->err('ERR_NULL_RESOURCE') + unless $self->get_workingresource(); + + # Check if they passed a reference to content rather than a filename. + my $content_ptr = ( ref($local) eq "SCALAR" ) ? 1 : 0; + + # Setup the resource based on the passed url + # Check if the remote resource exists and is a collection. + $url = $self->get_absolute_uri($url); + my $resource = $self->new_resource($url); + my $response = $resource->propfind( -depth => 0 ); + my $leaf_name; + if ( $response->is_success && $resource->is_collection && !$content_ptr ) + { + + # Add one / to the end of the collection + $url =~ s/\/*$//g; #Strip em + $url .= "/"; #Add one + $leaf_name = HTTP::DAV::Utils::get_leafname($local); + } + else { + $leaf_name = HTTP::DAV::Utils::get_leafname($url); + } + + my $target = $self->get_absolute_uri( $leaf_name, $url ); + + print "$local => $target ($url, $leaf_name)\n"; + + # PUT A DIRECTORY + if ( !$content_ptr && -d $local ) { + + # mkcol + # Return 0 if fail because the error will have already + # been set by the mkcol routine + if ( $self->mkcol($target, -headers => $custom_headers) ) { + if ( !opendir( DIR, $local ) ) { + $self->err( 'ERR_GENERIC', "chdir to \"$local\" failed: $!" ); + } + else { + my @files = readdir(DIR); + close DIR; + foreach my $file (@files) { + next if $file eq "."; + next if $file eq ".."; + my $progeny = "$local/$file"; + $progeny =~ s#//#/#g; # Fold down double slashes + $self->_put( + -local => $progeny, + -url => "$target/$file", + ); + } + } + } + + # PUT A FILE + } + else { + my $content = ""; + my $fail = 0; + if ($content_ptr) { + $content = $$local; + } + else { + if ( !CORE::open( F, $local ) ) { + $self->err( 'ERR_GENERIC', + "Couldn't open local file $local: $!" ); + $fail = 1; + } + else { + binmode F; + while () { $content .= $_; } + close F; + } + } + + if ( !$fail ) { + my $resource = $self->new_resource( -uri => $target ); + my $response = $resource->put($content,$custom_headers); + if ( $response->is_success ) { + $self->ok( "put $target (" . length($content) . " bytes)", + $target ); + } + else { + $self->err( 'ERR_RESP_FAIL', + "put failed " . $response->message(), $target ); + } + } + } +} + +###################################################################### +# UTILITY FUNCTION +# get_absolute_uri: +# Synopsis: $new_url = get_absolute_uri("/foo/bar") +# Takes a URI (or string) +# and returns the absolute URI based +# on the remote current working directory +sub get_absolute_uri { + my ( $self, @p ) = @_; + my ( $rel_uri, $base_uri ) + = HTTP::DAV::Utils::rearrange( [ 'REL_URI', 'BASE_URI' ], @p ); + + local $URI::URL::ABS_REMOTE_LEADING_DOTS = 1; + if ( !defined $base_uri ) { + $base_uri = $self->get_workingresource()->get_uri(); + } + + if ($base_uri) { + my $new_url = URI->new_abs( $rel_uri, $base_uri ); + return $new_url; + } + else { + $rel_uri; + } +} + +## Takes a $dav->get_globs(URI) +# Where URI may contain wildcards at the leaf level: +# URI: +# http://www.host.org/perldav/test*.html +# /perldav/test?.html +# test[12].html +# +# Performs a propfind to determine the url's that match +# +sub get_globs { + my ( $self, $url ) = @_; + my @urls = (); + my ( $left, $leafname ) = HTTP::DAV::Utils::split_leaf($url); + + # We need to unescape it because it may have been encoded. + $leafname = URI::Escape::uri_unescape($leafname); + + if ( $leafname =~ /[\*\?\[]/ ) { + my $resource = $self->new_resource( -uri => $left ); + my $resp = $resource->propfind( -depth => 1 ); + if ( $resp->is_error ) { + $self->err( 'ERR_RESP_FAIL', $resp->message(), $left ); + return (); + } + + $leafname = HTTP::DAV::Utils::glob2regex($leafname); + my $rl = $resource->get_resourcelist(); + if ($rl) { + my $match = 0; + + # We eval this because a bogus leafname could bomb the regex. + eval { + foreach my $progeny ( $rl->get_resources() ) + { + my $progeny_url = $progeny->get_uri; + my $progeny_leaf + = HTTP::DAV::Utils::get_leafname($progeny_url); + if ( $progeny_leaf =~ /^$leafname$/ ) { + print "Matched $progeny_url\n" + if $HTTP::DAV::DEBUG > 1; + $match++; + push( @urls, $progeny_url ); + } + else { + print "Skipped $progeny_url\n" + if $HTTP::DAV::DEBUG > 1; + } + } + }; + $self->err( 'ERR_GENERIC', "No match found" ) unless ($match); + } + } + else { + push( @urls, $url ); + } + + return @urls; +} + +sub what_happened { + my ($self, $url, $resource, $response) = @_; + + if (! $response->is_error()) { + return { success => 1 } + } + + my $error_type; + my $error_msg; + + # Method not allowed + if ($response->status_line =~ m{405}) { + $error_type = 'ERR_405'; + $error_msg = $response->status_line; + } + # 501 most probably means your LWP doesn't support SSL + elsif ($response->status_line =~ m{501}) { + $error_type = 'ERR_501'; + $error_msg = $response->status_line; + } + elsif ($response->www_authenticate) { + $error_type = 'ERR_UNAUTHORIZED'; + $error_msg = $response->www_authenticate; + } + elsif ( !$resource->is_dav_compliant ) { + $error_type = 'ERR_GENERIC'; + $error_msg = qq{The URL "$url" is not DAV enabled or not accessible.}; + } + else { + $error_type = 'ERR_RESP_FAIL'; + my $message = $response->message(); + $error_msg = qq{Could not access $url: $message}; + } + + return { + success => 0, + error_type => $error_type, + error_msg => $error_msg, + } + +} + +1; + +__END__ + + +=head1 NAME + +HTTP::DAV - A WebDAV client library for Perl5 + +=head1 SYNOPSIS + + # DAV script that connects to a webserver, safely makes + # a new directory and uploads all html files in + # the /tmp directory. + + use HTTP::DAV; + + $d = HTTP::DAV->new(); + $url = "http://host.org:8080/dav/"; + + $d->credentials( + -user => "pcollins", + -pass => "mypass", + -url => $url, + -realm => "DAV Realm" + ); + + $d->open( -url => $url ) + or die("Couldn't open $url: " .$d->message . "\n"); + + # Make a null lock on newdir + $d->lock( -url => "$url/newdir", -timeout => "10m" ) + or die "Won't put unless I can lock for 10 minutes\n"; + + # Make a new directory + $d->mkcol( -url => "$url/newdir" ) + or die "Couldn't make newdir at $url\n"; + + # Upload multiple files to newdir. + if ( $d->put( -local => "/tmp/*.html", -url => $url ) ) { + print "successfully uploaded multiple files to $url\n"; + } else { + print "put failed: " . $d->message . "\n"; + } + + $d->unlock( -url => $url ); + +=head1 DESCRIPTION + +HTTP::DAV is a Perl API for interacting with and modifying content on webservers using the WebDAV protocol. Now you can LOCK, DELETE and PUT files and much more on a DAV-enabled webserver. + +HTTP::DAV is part of the PerlDAV project hosted at http://www.webdav.org/perldav/ and has the following features: + +=over 4 + +=item * + +Full RFC2518 method support. OPTIONS, TRACE, GET, HEAD, DELETE, PUT, COPY, MOVE, PROPFIND, PROPPATCH, LOCK, UNLOCK. + +=item * + +A fully object-oriented API. + +=item * + +Recursive GET and PUT for site backups and other scripted transfers. + +=item * + +Transparent lock handling when performing LOCK/COPY/UNLOCK sequences. + +=item * + +http and https support (https requires the Crypt::SSLeay library). See INSTALLATION. + +=item * + +Basic AND Digest authentication support (Digest auth requires the MD5 library). See INSTALLATION. + +=item * + +C, a fully-functional ftp-style interface written on top of the HTTP::DAV API and bundled by default with the HTTP::DAV library. (If you've already installed HTTP::DAV, then dave will also have been installed (probably into /usr/local/bin). You can see it's man page by typing "perldoc dave" or going to http://www.webdav.org/perldav/dave/. + +=item * + +It is built on top of the popular LWP (Library for WWW access in Perl). This means that HTTP::DAV inherits proxy support, redirect handling, basic (and digest) authorization and many other HTTP operations. See C for more information. + +=item * + +Popular server support. HTTP::DAV has been tested against the following servers: mod_dav, IIS5, Xythos webfile server and mydocsonline. The library is growing an impressive interoperability suite which also serves as useful "sample scripts". See "make test" and t/*. + +=back + +C essentially has two API's, one which is accessed through this module directly (HTTP::DAV) and is a simple abstraction to the rest of the HTTP::DAV::* Classes. The other interface consists of the HTTP::DAV::* classes which if required allow you to get "down and dirty" with your DAV and HTTP interactions. + +The methods provided in C should do most of what you want. If, however, you need more control over the client's operations or need more info about the server's responses then you will need to understand the rest of the HTTP::DAV::* interfaces. A good place to start is with the C and C documentation. + +=head1 METHODS + +=head2 METHOD CALLING: Named vs Unnamed parameters + +You can pass parameters to C methods in one of two ways: named or unnamed. + +Named parameters provides for a simpler/easier to use interface. A named interface affords more readability and allows the developer to ignore a specific order on the parameters. (named parameters are also case insensitive) + +Each argument name is preceded by a dash. Neither case nor order matters in the argument list. -url, -Url, and -URL are all acceptable. In fact, only the first argument needs to begin with a dash. If a dash is present in the first argument, C assumes dashes for the subsequent ones. + +Each method can also be called with unnamed parameters which often makes sense for methods with only one parameter. But the developer will need to ensure that the parameters are passed in the correct order (as listed in the docs). + + Doc: method( -url=>$url, [-depth=>$depth] ) + Named: $d->method( -url=>$url, -depth=>$d ); # VALID + Named: $d->method( -Depth=>$d, -Url=>$url ); # VALID + Named: $d->method( Depth=>$d, Url=>$url ); # INVALID (needs -) + Named: $d->method( -Arg2=>$val2 ); # INVALID, ARG1 is not optional + Unnamed: $d->method( $val1 ); # VALID + Unnamed: $d->method( $val2,$val1 ); # INVALID, ARG1 must come first. + +IMPORTANT POINT!!!! If you specify a named parameter first but then forget for the second and third parameters, you WILL get weird things happen. E.g. this is bad: + + $d->method( -url=>$url, $arg2, $arg3 ); # BAD BAD BAD + +=head2 THINGS YOU NEED TO KNOW + +In all of the methods specified in L there are some common concepts you'll need to understand: + +=over 4 + +=item * URLs represent an absolute or relative URI. + + -url=>"host.org/dav_dir/" # Absolute + -url=>"/dav_dir/" # Relative + -url=>"file.txt" # Relative + +You can only use a relative URL if you have already "open"ed an absolute URL. + +The HTTP::DAV module now consistently uses the named parameter: URL. The lower-level HTTP::DAV::Resource interface inconsistently interchanges URL and URI. I'm working to resolve this, in the meantime, you'll just need to remember to use the right one by checking the documentation if you need to mix up your use of both interfaces. + +=item * GLOBS + +Some methods accept wildcards in the URL. A wildcard can be used to indicate that the command should perform the command on all Resources that match the wildcard. These wildcards are called GLOBS. + +The glob may contain the characters "*", "?" and the set operator "[...]" where ... contains multiple characters ([1t2]) or a range such ([1-5]). For the curious, the glob is converted to a regex and then matched: "*" to ".*", "?" to ".", and the [] is left untouched. + +It is important to note that globs only operate at the leaf-level. For instance "/my_dir/*/file.txt" is not a valid glob. + +If a glob matches no URL's the command will fail (which normally means returns 0). + +Globs are useful in conjunction with L to provide feedback as each operation completes. + +See the documentation for each method to determine whether it supports globbing. + +Globs are useful for interactive style applications (see the source code for C as an example). + +Example globs: + + $dav1->delete(-url=>"/my_dir/file[1-3]"); # Matches file1, file2, file3 + $dav1->delete(-url=>"/my_dir/file[1-3]*.txt");# Matches file1*.txt,file2*.txt,file3*.txt + $dav1->delete(-url=>"/my_dir/*/file.txt"); # Invalid. Can only match at leaf-level + +=item * CALLBACKS + +Callbacks are used by some methods (primarily get and put) to give the caller some insight as to how the operation is progressing. A callback allows you to define a subroutine as defined below and pass a reference (\&ref) to the method. + +The rationale behind the callback is that a recursive get/put or an operation against many files (using a C) can actually take a long time to complete. + +Example callback: + + $d->get( -url=>$url, -to=>$to, -callback=>\&mycallback ); + +Your callback function MUST accept arguments as follows: + sub cat_callback { + my($status,$mesg,$url,$so_far,$length,$data) = @_; + ... + } + +The C argument specifies whether the operation has succeeded (1), failed (0), or is in progress (-1). + +The C argument is a status message. The status message could contain any string and often contains useful error messages or success messages. + +The C the remote URL. + +The C, C - these parameters indicate how many bytes have been downloaded and how many we should expect. This is useful for doing "56% to go" style-gauges. + +The C parameter - is the actual data transferred. The C command uses this to print the data to the screen. This value will be empty for C. + +See the source code of C for a useful sample of how to setup a callback. + +Note that these arguments are NOT named parameters. + +All error messages set during a "multi-operation" request (for instance a recursive get/put) are also retrievable via the C function once the operation has completed. See C for more information. + +=back + +=head2 PUBLIC METHODS + +=over 4 + +=item B + +=item B + +Creates a new C client + + $d = HTTP::DAV->new(); + +The C<-useragent> parameter allows you to pass your own B and expects an C object. See the C program for an advanced example of a custom UserAgent that interactively prompts the user for their username and password. + +The C<-headers> parameter allows you to specify a list of headers to be sent along with all requests. This can be either a hashref like: + + { "X-My-Header" => "value", ... } + +or a L object. + +=item B + +sets authorization credentials for a C and/or C. + +When the client hits a protected resource it will check these credentials to see if either the C or C match the authorization response. + +Either C or C must be provided. + +returns no value + +Example: + + $d->credentials( -url=>'myhost.org:8080/test/', + -user=>'pcollins', + -pass=>'mypass'); + +=item B + +sets the debug level to C<$val>. 0=off 3=noisy. + +C<$val> default is 0. + +returns no value. + +When the value is greater than 1, the C module will log all of the client<=>server interactions into /tmp/perldav_debug.txt. + +=back + +=head2 DAV OPERATIONS + +For all of the following operations, URL can be absolute (http://host.org/dav/) or relative (../dir2/). The only operation that requires an absolute URL is open. + +=over 4 + +=item B + +copies one remote resource to another + +=over 4 + +=item C<-url> + +is the remote resource you'd like to copy. Mandatory + +=item C<-dest> + +is the remote target for the copy command. Mandatory + +=item C<-overwrite> + +optionally indicates whether the server should fail if the target exists. Valid values are "T" and "F" (1 and 0 are synonymous). Default is T. + +=item C<-depth> + +optionally indicates whether the server should do a recursive copy or not. Valid values are 0 and (1 or "infinity"). Default is "infinity" (1). + +=back + +The return value is always 1 or 0 indicating success or failure. + +Requires a working resource to be set before being called. See C. + +Note: if either C<'URL'> or C<'DEST'> are locked by this dav client, then the lock headers will be taken care of automatically. If the either of the two URL's are locked by someone else, the server should reject the request. + +B + + $d->open(-url=>"host.org/dav_dir/"); + +Recursively copy dir1/ to dir2/ + + $d->copy(-url=>"dir1/", -dest=>"dir2/"); + +Non-recursively and non-forcefully copy dir1/ to dir2/ + + $d->copy(-url=>"dir1/", -dest=>"dir2/",-overwrite=>0,-depth=>0); + +Create a copy of dir1/file.txt as dir2/file.txt + + $d->cwd(-url=>"dir1/"); + $d->copy("file.txt","../dir2"); + +Create a copy of file.txt as dir2/new_file.txt + + $d->copy("file.txt","/dav_dir/dir2/new_file.txt") + +=item B + +changes the remote working directory. + +This is synonymous to open except that the URL can be relative and may contain a C (the first match in a glob will be used). + + $d->open("host.org/dav_dir/dir1/"); + $d->cwd("../dir2"); + $d->cwd(-url=>"../dir1"); + +The return value is always 1 or 0 indicating success or failure. + +Requires a working resource to be set before being called. See C. + +You can not cwd to files, only collections (directories). + +=item B + +deletes a remote resource. + + $d->open("host.org/dav_dir/"); + $d->delete("index.html"); + $d->delete("./dir1"); + $d->delete(-url=>"/dav_dir/dir2/file*",-callback=>\&mycallback); + +=item C<-url> + +is the remote resource(s) you'd like to delete. It can be a file, directory or C. + +=item C<-callback> is a reference to a callback function which will be called everytime a file is deleted. This is mainly useful when used in conjunction with L deletes. See L + +The return value is always 1 or 0 indicating success or failure. + +Requires a working resource to be set before being called. See C. + +This command will recursively delete directories. BE CAREFUL of uninitialised file variables in situation like this: $d->delete("$dir/$file"). This will trash your $dir if $file is not set. + +=item B + +downloads the file or directory at C to the local location indicated by C. + +=over 4 + +=item C<-url> + +is the remote resource you'd like to get. It can be a file or directory or a "glob". + +=item C<-to> + +is where you'd like to put the remote resource. The -to parameter can be: + + - a B indicating where to save the contents. + + - a B. + + - a reference to a B into which the contents will be saved. + +If the C<-url> matches multiple files (via a glob or a directory download), then the C routine will return an error if you try to use a FileHandle reference or a scalar reference. + +=item C<-callback> + +is a reference to a callback function which will be called everytime a file is completed downloading. The idea of the callback function is that some recursive get's can take a very long time and the user may require some visual feedback. See L for an examples and how to use a callback. + +=back + +The return value of get is always 1 or 0 indicating whether the entire get sequence was a success or if there was ANY failures. For instance, in a recursive get, if the server couldn't open 1 of the 10 remote files, for whatever reason, then the return value will be 0. This is so that you can have your script call the C routine to handle error conditions. + +Previous versions of HTTP::DAV allowed the return value to be the file contents if no -to attribute was supplied. This functionality is deprecated. + +Requires a working resource to be set before being called. See C. + +B + + $d->open("host.org/dav_dir/"); + +Recursively get remote my_dir/ to . + + $d->get("my_dir/","."); + +Recursively get remote my_dir/ to /tmp/my_dir/ calling &mycallback($success,$mesg) everytime a file operation is completed. + + $d->get("my_dir","/tmp",\&mycallback); + +Get remote my_dir/index.html to /tmp/index.html + + $d->get(-url=>"/dav_dir/my_dir/index.html",-to=>"/tmp"); + +Get remote index.html to /tmp/index1.html + + $d->get("index.html","/tmp/index1.html"); + +Get remote index.html to a filehandle + + my $fh = new FileHandle; + $fh->open(">/tmp/index1.html"); + $d->get("index.html",\$fh); + +Get remote index.html as a scalar (into the string $file_contents): + + my $file_contents; + $d->get("index.html",\$file_contents); + +Get all of the files matching the globs file1* and file2*: + + $d->get("file[12]*","/tmp"); + +Get all of the files matching the glob file?.html: + + $d->get("file?.html","/tmp"); # downloads file1.html and file2.html but not file3.html or file1.txt + +Invalid glob: + + $d->get("/dav_dir/*/index.html","/tmp"); # Can not glob like this. + +=item B + +locks a resource. If URL is not specified, it will lock the current working resource (opened resource). + + $d->lock( -url => "index.html", + -owner => "Patrick Collins", + -depth => "infinity", + -scope => "exclusive", + -type => "write", + -timeout => "10h" ) + +See C lock() for details of the above parameters. + +The return value is always 1 or 0 indicating success or failure. + +Requires a working resource to be set before being called. See C. + +When you lock a resource, the lock is held against the current HTTP::DAV object. In fact, the locks are held in a C object. You can operate against all of the locks that you have created as follows: + + ## Print and unlock all locks that we own. + my $rl_obj = $d->get_lockedresourcelist(); + foreach $resource ( $rl_obj->get_resources() ) { + @locks = $resource->get_locks(-owned=>1); + foreach $lock ( @locks ) { + print $resource->get_uri . "\n"; + print $lock->as_string . "\n"; + } + ## Unlock them? + $resource->unlock; + } + +Typically, a simple $d->unlock($uri) will suffice. + +B + + $d->lock($uri, -timeout=>"1d"); + ... + $d->put("/tmp/index.html",$uri); + $d->unlock($uri); + +=item B + +make a remote collection (directory) + +The return value is always 1 or 0 indicating success or failure. + +Requires a working resource to be set before being called. See C. + + $d->open("host.org/dav_dir/"); + $d->mkcol("new_dir"); # Should succeed + $d->mkcol("/dav_dir/new_dir"); # Should succeed + $d->mkcol("/dav_dir/new_dir/xxx/yyy"); # Should fail + +=item B + +moves one remote resource to another + +=over 4 + +=item C<-url> + +is the remote resource you'd like to move. Mandatory + +=item C<-dest> + +is the remote target for the move command. Mandatory + +=item C<-overwrite> + +optionally indicates whether the server should fail if the target exists. Valid values are "T" and "F" (1 and 0 are synonymous). Default is T. + +=back + +Requires a working resource to be set before being called. See C. + +The return value is always 1 or 0 indicating success or failure. + +Note: if either C<'URL'> or C<'DEST'> are locked by this dav client, then the lock headers will be taken care of automatically. If either of the two URL's are locked by someone else, the server should reject the request. + +B + + $d->open(-url=>"host.org/dav_dir/"); + +move dir1/ to dir2/ + + $d->move(-url=>"dir1/", -dest=>"dir2/"); + +non-forcefully move dir1/ to dir2/ + + $d->move(-url=>"dir1/", -dest=>"dir2/",-overwrite=>0); + +Move dir1/file.txt to dir2/file.txt + + $d->cwd(-url=>"dir1/"); + $d->move("file.txt","../dir2"); + +move file.txt to dir2/new_file.txt + + $d->move("file.txt","/dav_dir/dir2/new_file.txt") + +=item B + +opens the directory (collection resource) at URL. + +open will perform a propfind against URL. If the server does not understand the request then the open will fail. + +Similarly, if the server indicates that the resource at URL is NOT a collection, the open command will fail. + +=item B + +Performs an OPTIONS request against the URL or the working resource if URL is not supplied. + +Requires a working resource to be set before being called. See C. + +The return value is a string of comma separated OPTIONS that the server states are legal for URL or undef otherwise. + +A fully compliant DAV server may offer as many methods as: OPTIONS, TRACE, GET, HEAD, DELETE, PUT, COPY, MOVE, PROPFIND, PROPPATCH, LOCK, UNLOCK + +Note: IIS5 does not support PROPPATCH or LOCK on collections. + +Example: + + $options = $d->options($url); + print $options . "\n"; + if ($options=~ /\bPROPPATCH\b/) { + print "OK to proppatch\n"; + } + +Or, put more simply: + + if ( $d->options($url) =~ /\bPROPPATCH\b/ ) { + print "OK to proppatch\n"; + } + +=item B + +Perform a propfind against URL at DEPTH depth. + +C<-depth> can be used to specify how deep the propfind goes. "0" is collection only. "1" is collection and it's immediate members (This is the default value). "infinity" is the entire directory tree. Note that most DAV compliant servers deny "infinity" depth propfinds for security reasons. + +Requires a working resource to be set before being called. See C. + +The return value is an C object on success or 0 on failure. + +The Resource object can be used for interrogating properties or performing other operations. + + ## Print collection or content length + if ( $r=$d->propfind( -url=>"/my_dir", -depth=>1) ) { + if ( $r->is_collection ) { + print "Collection\n" + print $r->get_resourcelist->as_string . "\n" + } else { + print $r->get_property("getcontentlength") ."\n"; + } + } + +Please note that although you may set a different namespace for a property of a resource during a set_prop, HTTP::DAV currently ignores all XML namespaces so you will get clashes if two properties have the same name but in different namespaces. Currently this is unavoidable but I'm working on the solution. + +=item B + +If C<-action> equals "set" then we set a property named C<-propname> to C<-propvalue> in the namespace C<-namespace> for C<-url>. + +If C<-action> equals "remove" then we unset a property named C<-propname> in the namespace C<-namespace> for C<-url>. + +If no action is supplied then the default action is "set". + +The return value is an C object on success or 0 on failure. + +The Resource object can be used for interrogating properties or performing other operations. + +To explicitly set a namespace in which to set the propname then you can use the C<-namespace> and C<-nsabbr> (namespace abbreviation) parameters. But you're welcome to play around with DAV namespaces. + +Requires a working resource to be set before being called. See C. + +It is recommended that you use C and C instead of proppatch for readability. + +C simply calls Cset)> and C calls C"remove")> + +See C and C for examples. + +=item B + +uploads the files or directories at C<-local> to the remote destination at C<-url>. + +C<-local> points to a file, directory or series of files or directories (indicated by a glob). + +If the filename contains any of the characters `*', `?' or `[' it is a candidate for filename substitution, also known as ``globbing''. This word is then regarded as a pattern (``glob-pattern''), and replaced with an alphabetically sorted list of file names which match the pattern. + +One can upload/put a string by passing a reference to a scalar in the -local parameter. See example below. + +put requires a working resource to be set before being called. See C. + +The return value is always 1 or 0 indicating success or failure. + +See L for a description of what the optional callback parameter does. + +You can also pass a C<-headers> argument. That allows to specify custom HTTP headers. It can be either a hashref with header names and values, or a L object. + +B + +Put a string to the server: + + my $myfile = "This is the contents of a file to be uploaded\n"; + $d->put(-local=>\$myfile,-url=>"http://www.host.org/dav_dir/file.txt"); + +Put a local file to the server: + + $d->put(-local=>"/tmp/index.html",-url=>"http://www.host.org/dav_dir/"); + +Put a series of local files to the server: + + In these examples, /tmp contains file1.html, file1, file2.html, + file2.txt, file3.html, file2/ + + $d->put(-local=>"/tmp/file[12]*",-url=>"http://www.host.org/dav_dir/"); + + uploads file1.html, file1, file2.html, file2.txt and the directory file2/ to dav_dir/. + +=item B + +Sets a property named C<-propname> to C<-propvalue> in the namespace C<-namespace> for C<-url>. + +Requires a working resource to be set before being called. See C. + +The return value is an C object on success or 0 on failure. + +The Resource object can be used for interrogating properties or performing other operations. + +Example: + + if ( $r = $d->set_prop(-url=>$url, + -namespace=>"dave", + -propname=>"author", + -propvalue=>"Patrick Collins" + ) ) { + print "Author property set\n"; + } else { + print "set_prop failed:" . $d->message . "\n"; + } + +See the note in propfind about namespace support in HTTP::DAV. They're settable, but not readable. + + + +=item B + +forcefully steals any locks held against URL. + +steal will perform a propfind against URL and then, any locks that are found will be unlocked one by one regardless of whether we own them or not. + +Requires a working resource to be set before being called. See C. + +The return value is always 1 or 0 indicating success or failure. If multiple locks are found and unlocking one of them fails then the operation will be aborted. + + if ($d->steal()) { + print "Steal succeeded\n"; + } else { + print "Steal failed: ". $d->message() . "\n"; + } + +=item B + +unlocks any of our locks on URL. + +Requires a working resource to be set before being called. See C. + +The return value is always 1 or 0 indicating success or failure. + + if ($d->unlock()) { + print "Unlock succeeded\n"; + } else { + print "Unlock failed: ". $d->message() . "\n"; + } + +=item B + +Unsets a property named C<-propname> in the namespace C<-namespace> for C<-url>. +Requires a working resource to be set before being called. See C. + +The return value is an C object on success or 0 on failure. + +The Resource object can be used for interrogating properties or performing other operations. + +Example: + + if ( $r = $d->unset_prop(-url=>$url, + -namespace=>"dave", + -propname=>"author", + ) ) { + print "Author property was unset\n"; + } else { + print "set_prop failed:" . $d->message . "\n"; + } + +See the note in propfind about namespace support in HTTP::DAV. They're settable, but not readable. + +=back + +=head2 ACCESSOR METHODS + +=over 4 + +=item B + +Returns the clients' working C object. + +You may want to interact with the C object +to modify request headers or provide advanced authentication +procedures. See dave for an advanced authentication procedure. + +=item B + +Takes no arguments and returns the clients' last outgoing C object. + +You would only use this to inspect a request that has already occurred. + +If you would like to modify the C BEFORE the HTTP request takes place (for instance to add another header), you will need to get the C using C and interact with that. + +=item B + +Returns the currently "opened" or "working" resource (C). + +The working resource is changed whenever you open a url or use the cwd command. + +e.g. + $r = $d->get_workingresource + print "pwd: " . $r->get_uri . "\n"; + +=item B + +Returns the currently "opened" or "working" C. + +The working resource is changed whenever you open a url or use the cwd command. + + print "pwd: " . $d->get_workingurl . "\n"; + +=item B + +Returns an C object that represents all of the locks we've created using THIS dav client. + + print "pwd: " . $d->get_workingurl . "\n"; + +=item B + +This is a useful utility function which joins C and C and returns a new URI. + +If C is not supplied then the current working resource (as indicated by get_workingurl) is used. If C is not set and there is no current working resource the C will be returned. + +For instance: + $d->open("http://host.org/webdav/dir1/"); + + # Returns "http://host.org/webdav/dir2/" + $d->get_absolute_uri(-rel_uri=>"../dir2"); + + # Returns "http://x.org/dav/dir2/file.txt" + $d->get_absolute_uri(-rel_uri =>"dir2/file.txt", + ->base_uri=>"http://x.org/dav/"); + +Note that it subtly takes care of trailing slashes. + +=back + +=head2 ERROR HANDLING METHODS + +=over 4 + +=item B + +C gets the last success or error message. + +The return value is always a scalar (string) and will change everytime a dav operation is invoked (lock, cwd, put, etc). + +See also C for operations which contain multiple error messages. + +=item B + +Returns an @array of error messages that had been set during a multi-request operation. + +Some of C's operations perform multiple request to the server. At the time of writing only put and get are considered multi-request since they can operate recursively requiring many HTTP requests. + +In these situations you should check the errors array if to determine if any of the requests failed. + +The C function is used for multi-request operations and not to be confused with a multi-status server response. A multi-status server response is when the server responds with multiple error messages for a SINGLE request. To deal with multi-status responses, see C. + + # Recursive put + if (!$d->put( "/tmp/my_dir", $url ) ) { + # Get the overall message + print $d->message; + # Get the individual messages + foreach $err ( $d->errors ) { print " Error:$err\n" } + } + +=item B + +Returns the status of the last DAV operation performed through the HTTP::DAV interface. + +This value will always be the same as the value returned from an HTTP::DAV::method. For instance: + + # This will always evaluate to true + ($d->lock($url) eq $d->is_success) ? + +You may want to use the is_success method if you didn't capture the return value immediately. But in most circumstances you're better off just evaluating as follows: + if($d->lock($url)) { ... } + +=item B + +Takes no arguments and returns the last seen C object. + +You may want to use this if you have just called a propfind and need the individual error messages returned in a MultiStatus. + +If you find that you're using get_last_response() method a lot, you may be better off using the more advanced C interface and interacting with the HTTP::DAV::* interfaces directly as discussed in the intro. For instance, if you find that you're always wanting a detailed understanding of the server's response headers or messages, then you're probably better off using the C methods and interpreting the C directly. + +To perform detailed analysis of the server's response (if for instance you got back a multistatus response) you can call C which will return to you the most recent response object (always the result of the last operation, PUT, PROPFIND, etc). With the returned HTTP::DAV::Response object you can handle multi-status responses. + +For example: + + # Print all of the messages in a multistatus response + if (! $d->unlock($url) ) { + $response = $d->get_last_response(); + if ($response->is_multistatus() ) { + foreach $num ( 0 .. $response->response_count() ) { + ($err_code,$mesg,$url,$desc) = + $response->response_bynum($num); + print "$mesg ($err_code) for $url\n"; + } + } + } + +=back + +=head2 ADVANCED METHODS + +=over 4 + +=item B + +Creates a new resource object with which to play. +This is the preferred way of creating an C object if required. +Why? Because each Resource object needs to sit within a global HTTP::DAV client. +Also, because the new_resource routine checks the C locked resource +list before creating a new object. + + $dav->new_resource( -uri => "http://..." ); + +=item B + +Sets the current working resource to URL. + +You shouldn't need this method. Call open or cwd to set the working resource. + +You CAN call C but you will need to perform a +C immediately following it to ensure that the working +resource is valid. + +=back + +=head1 INSTALLATION, TODO, MAILING LISTS and REVISION HISTORY + +[OUTDATED] + +Please see the primary HTTP::DAV webpage at +(http://www.webdav.org/perldav/http-dav/) +or the README file in this library. + +=head1 SEE ALSO + +You'll want to also read: + +=over * + +=item C + +=item C + +=item C + +=back + +and maybe if you're more inquisitive: + +=over * + +=item C + +=item C + +=item C + +=item C + +=item C + +=item C + +=back + +=head1 AUTHOR AND COPYRIGHT + +This module is Copyright (C) 2001-2008 by + + Patrick Collins + G03 Gloucester Place, Kensington + Sydney, Australia + + Email: pcollins@cpan.org + Phone: +61 2 9663 4916 + +All rights reserved. + +Current co-maintainer of the module is Cosimo Streppone +for Opera Software ASA, L. + +You may distribute this module under the terms of either the +GNU General Public License or the Artistic License, +as specified in the Perl README file. + +=cut diff --git a/tests/ownCloud/mocka/CMakeLists.txt b/tests/ownCloud/mocka/CMakeLists.txt index f329d0e61c..3383478503 100644 --- a/tests/ownCloud/mocka/CMakeLists.txt +++ b/tests/ownCloud/mocka/CMakeLists.txt @@ -13,8 +13,6 @@ include_directories( add_executable(ocmod_test ocmod_test.c) -add_executable(md5_test md5_test.c) target_link_libraries(ocmod_test ${CMOCKA_LIBRARIES} ${NEON_LIBRARIES} ${CSYNC_LIBRARY} ) -target_link_libraries(md5_test ${CMOCKA_LIBRARIES} ${CSYNC_LIBRARY} ) diff --git a/tests/ownCloud/mocka/md5_test.c b/tests/ownCloud/mocka/md5_test.c deleted file mode 100644 index 1975b1576c..0000000000 --- a/tests/ownCloud/mocka/md5_test.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2008 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#define _GNU_SOURCE /* See feature_test_macros(7) */ -#include - -#include -#include -#include -#include -#include - -#include - -#include - -#include "config_test.h" - - -static void test_md5_buffer(void **state) { - const char *t1 = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet."; - const char *t2 = "This is a nice md5 test"; - const char *t3 = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." - "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet."; - - (void) state; - assert_string_equal( csync_buffer_md5(t1, strlen(t1)), "dbcb1ed05ecf975f532604f2d2000246"); - assert_string_equal( csync_buffer_md5(t2, strlen(t2)), "65236ae09754bca371fb869384040141"); - assert_string_equal( csync_buffer_md5(t3, strlen(t3)), "651f37892a9df60ed087bc7a1c660fec"); -} - -static void test_md5_files(void **state) { - char path[255]; - (void) state; - - strcpy(path, TESTFILES_DIR); - strcat(path, "test.txt"); - assert_string_equal( csync_file_md5(path), "f3971ce599093756e6018513d0835134"); - - strcpy(path, TESTFILES_DIR); - strcat(path, "red_is_the_rose.jpg"); - assert_string_equal( csync_file_md5(path), "baf8eeb2a36af94f033fa0094c50c2d5"); - -} - - -int main(void) { - const UnitTest tests[] = { - unit_test(test_md5_buffer), - unit_test(test_md5_files) - }; - - return run_tests(tests); -} diff --git a/tests/ownCloud/mocka/ocmod_test.c b/tests/ownCloud/mocka/ocmod_test.c index 74365c69cc..77104ed86d 100644 --- a/tests/ownCloud/mocka/ocmod_test.c +++ b/tests/ownCloud/mocka/ocmod_test.c @@ -1,17 +1,15 @@ /* - * Copyright 2008 Google Inc. + * Copyright 2012 Klaas Freitag * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. */ @@ -99,7 +97,7 @@ static void connect_test_success(void **state) { (void) state; strcpy(buf, TEST_CONFIG_DIR); - strcat(buf, "test.cfg"); + strcat(buf, "mockatest.cfg"); assert_true( load_oc_config( buf )); @@ -134,6 +132,7 @@ static void fetch_a_context(void **state) { assert_true( fetchCtx->currResource->name != NULL ); printf( " %s -> %s\n", fetchCtx->currResource->uri, fetchCtx->currResource->name ); + printf( " MD5: %s\n", fetchCtx->currResource->md5); fetchCtx->currResource = fetchCtx->currResource->next; } } @@ -299,11 +298,13 @@ int main(void) { const UnitTest tests[] = { unit_test(null_test_success), unit_test(connect_test_success), - unit_test(fetch_a_context), unit_test_setup_teardown(test_setup_dirs, setup_toplevel_dir, teardown_toplevel_dir), unit_test(test_upload_files), + unit_test(fetch_a_context), unit_test(test_download_files), }; + srand(time(NULL)); + return run_tests(tests); } diff --git a/tests/ownCloud/mockatest.cfg.in b/tests/ownCloud/mockatest.cfg.in new file mode 100644 index 0000000000..8308db6abd --- /dev/null +++ b/tests/ownCloud/mockatest.cfg.in @@ -0,0 +1,6 @@ +[global] + +user = 'joe' +pwd = 'secret' +host = 'localhost/owncloud' + diff --git a/tests/ownCloud/t1.pl b/tests/ownCloud/t1.pl index 56191cd60c..8e1c5b46b5 100755 --- a/tests/ownCloud/t1.pl +++ b/tests/ownCloud/t1.pl @@ -7,6 +7,8 @@ # Copyright (C) by Klaas Freitag # +use lib "."; + use Carp::Assert; use Data::Dumper; use HTTP::DAV; @@ -110,7 +112,7 @@ sub csync( $$ ) my $cmd = "LD_LIBRARY_PATH=$ld_libpath $csync $local $url"; print "Starting: $cmd\n"; - system( $cmd ); + system( $cmd ) == 0 or die("CSync died!\n"); } # @@ -192,9 +194,11 @@ sub glob_put( $$$ ) foreach my $lfile( @puts ) { if( $lfile =~ /.*\/(.+)$/g ) { my $rfile = $1; - my $puturl = "$target/$rfile"; + my $puturl = "$target"."$rfile"; print " *** Putting $lfile to $puturl\n"; - $d->put( -local=>$lfile, -url=> $puturl ); + if( ! $d->put( -local=>$lfile, -url=> $puturl ) ) { + print " ### FAILED to put: ". $d->message . '\n'; + } } } } @@ -205,40 +209,51 @@ my $d = HTTP::DAV->new(); $d->credentials( -url=> $owncloud, -realm=>"ownCloud", -user=> $user, -pass=> $passwd ); -$d->DebugLevel(3); +# $d->DebugLevel(3); -my $remoteDir = "t1/"; +my $remoteDir = sprintf( "t1-%#.3o/", rand(1000) ); +print "Working in remote dir $remoteDir\n"; remoteDir( $d, $remoteDir ); # $d->open("localhost/oc/files/webdav.php/t1"); +print "Copy some files to the remote location\n"; remoteDir( $d, $remoteDir . "remoteToLocal1" ); +remoteDir( $d, $remoteDir . "remoteToLocal1/rtl1"); +remoteDir( $d, $remoteDir . "remoteToLocal1/rtl1/rtl11"); +remoteDir( $d, $remoteDir . "remoteToLocal1/rtl2"); # put some files remote. glob_put( $d, 'toremote1/*', $owncloud . $remoteDir . "remoteToLocal1/" ); +glob_put( $d, 'toremote1/rtl1/*', $owncloud . $remoteDir . "remoteToLocal1/rtl1/" ); +glob_put( $d, 'toremote1/rtl1/rtl11/*', $owncloud . $remoteDir . "remoteToLocal1/rtl1/rtl11/" ); +glob_put( $d, 'toremote1/rtl2/*', $owncloud . $remoteDir . "remoteToLocal1/rtl2/" ); my $localDir = "./t1"; +print "Create the local sync dir $localDir\n"; createLocalDir( $localDir ); # call csync, sync local t1 to remote t1 csync( $localDir, $remoteDir ); -print "\nNow assertions:\n"; - # Check if the files from toremote1 are now in t1/remoteToLocal1 # they should have taken the way via the ownCloud. +print "Assert the local file copy\n"; assertLocalDirs( "./toremote1", "$localDir/remoteToLocal1" ); # Check if the synced files from ownCloud have the same timestamp as the local ones. +print "\nNow assert remote 'toremote1' with local \"$localDir/remoteToLocal1\" :\n"; assertLocalAndRemoteDir( $d, "$localDir/remoteToLocal1", $remoteDir . "remoteToLocal1" ); # remove a local file. +print "\nRemove a local file\n"; unlink( "$localDir/remoteToLocal1/kernelcrash.txt" ); csync( $localDir, $remoteDir ); assertLocalAndRemoteDir( $d, "$localDir/remoteToLocal1", $remoteDir . "remoteToLocal1" ); # add local files to a new dir1 +print "\nAdd some more files to local:\n"; my $locDir = $localDir . "/fromLocal1"; mkdir( $locDir ); @@ -248,15 +263,30 @@ foreach my $file ( <./tolocal1/*> ) { copy( $file, $locDir ); } csync( $localDir, $remoteDir ); +print "\nAssert local and remote dirs.\n"; assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" ); # move a local file +print "\nMove a file locally.\n"; move( "$locDir/kramer.jpg", "$locDir/oldtimer.jpg" ); csync( $localDir, $remoteDir ); assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" ); +# move a local directory. +print "\nMove a local directory.\n"; +move( "$localDir/remoteToLocal1/rtl1", "$localDir/remoteToLocal1/rtlX"); +csync( $localDir, $remoteDir ); +assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" ); + +# remove a local dir +print "\nRemove a local directory.\n"; +localCleanup( "$localDir/remoteToLocal1/rtlX" ); +csync( $localDir, $remoteDir ); +assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" ); + + print "\n###########################################\n"; -print " all cool - tests succeeded.\n"; +print " all cool - tests succeeded in $remoteDir.\n"; print "###########################################\n"; print "\nInterrupt before cleanup in 4 seconds...\n"; diff --git a/tests/ownCloud/testfiles/red_is_the_rose.jpg b/tests/ownCloud/testfiles/red_is_the_rose.jpg new file mode 100644 index 0000000000..4100de3f53 Binary files /dev/null and b/tests/ownCloud/testfiles/red_is_the_rose.jpg differ diff --git a/tests/ownCloud/testfiles/test.txt b/tests/ownCloud/testfiles/test.txt new file mode 100644 index 0000000000..f4586ae5b2 --- /dev/null +++ b/tests/ownCloud/testfiles/test.txt @@ -0,0 +1,11 @@ +Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. + +Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. + +Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. + +Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. + +Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis. + +At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur diff --git a/tests/ownCloud/toremote1/rtl1/La cédille b/tests/ownCloud/toremote1/rtl1/La cédille new file mode 100644 index 0000000000..055dab2542 --- /dev/null +++ b/tests/ownCloud/toremote1/rtl1/La cédille @@ -0,0 +1 @@ +A new text. diff --git a/tests/ownCloud/toremote1/rtl11/quitte.pdf b/tests/ownCloud/toremote1/rtl11/quitte.pdf new file mode 100644 index 0000000000..df5495f0a5 Binary files /dev/null and b/tests/ownCloud/toremote1/rtl11/quitte.pdf differ diff --git a/tests/ownCloud/toremote1/rtl11/red_is_the_rose.jpg b/tests/ownCloud/toremote1/rtl11/red_is_the_rose.jpg new file mode 100644 index 0000000000..4100de3f53 Binary files /dev/null and b/tests/ownCloud/toremote1/rtl11/red_is_the_rose.jpg differ diff --git a/tests/ownCloud/toremote1/rtl2/kb1.jpg b/tests/ownCloud/toremote1/rtl2/kb1.jpg new file mode 100644 index 0000000000..f10585c8a6 Binary files /dev/null and b/tests/ownCloud/toremote1/rtl2/kb1.jpg differ diff --git a/tests/ownCloud/toremote1/rtl2/émettre.xls b/tests/ownCloud/toremote1/rtl2/émettre.xls new file mode 100644 index 0000000000..ee3e603787 --- /dev/null +++ b/tests/ownCloud/toremote1/rtl2/émettre.xls @@ -0,0 +1 @@ +some content.