mirrors/nextcloud-desktop
3
0
Fork 0
mirror of https://github.com/nextcloud/desktop.git synced 2025-10-26 11:17:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fastsync' into dav

Browse Source
This commit is contained in:
Klaas Freitag 2012-10-01 10:35:28 +02:00
commit b25d77caee
41 changed files with 3591 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -8,11 +8,11 @@ set(APPLICATION_NAME ${PROJECT_NAME})
set(APPLICATION_VERSION_MAJOR "0")
set(APPLICATION_VERSION_MINOR "50")
set(APPLICATION_VERSION_PATCH "8")
set(APPLICATION_VERSION_PATCH "10")
set(APPLICATION_VERSION "${APPLICATION_VERSION_MAJOR}.${APPLICATION_VERSION_MINOR}.${APPLICATION_VERSION_PATCH}")
set(LIBRARY_VERSION "0.1.8")
set(LIBRARY_VERSION "0.1.10")
set(LIBRARY_SOVERSION "0")
# where to look first for cmake modules, before ${CMAKE_ROOT}/Modules/ is checked

2
CPackConfig.cmake
View File

@ -12,7 +12,7 @@ set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/COPYING")
### versions
set(CPACK_PACKAGE_VERSION_MAJOR "0")
set(CPACK_PACKAGE_VERSION_MINOR "49")
set(CPACK_PACKAGE_VERSION_MINOR "50")
set(CPACK_PACKAGE_VERSION_PATCH "9")
set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}")

306
modules/csync_owncloud.c
View File

@ -82,6 +82,7 @@ typedef unsigned long dav_size_t;
/* Struct to store data for each resource found during an opendir operation.
* It represents a single file entry.
*/
typedef struct resource {
char *uri; /* The complete uri */
char *name; /* The filename only */
@ -89,6 +90,7 @@ typedef struct resource {
enum resource_type type;
dav_size_t size;
time_t modtime;
char* md5;
struct resource *next;
} resource;
@ -117,6 +119,8 @@ struct transfer_context {
const char *method; /* the HTTP method, either PUT or GET */
ne_decompress *decompress; /* the decompress context */
int fileWritten; /* flag to indicate that a buffer file was written for PUTs */
char *md5;
char *clean_uri;
};
/* Struct with the WebDAV session */
@ -131,10 +135,12 @@ struct dav_session_s {
char *proxy_user;
char *proxy_pwd;
time_t prev_delta;
time_t time_delta; /* The time delta to use. */
char *session_key;
long int prev_delta;
long int time_delta; /* The time delta to use. */
long int time_delta_sum; /* What is the time delta average? */
int time_delta_cnt; /* How often was the server time gathered? */
long int time_delta_cnt; /* How often was the server time gathered? */
};
/* The list of properties that is fetched in PropFind on a collection */
@ -142,6 +148,7 @@ static const ne_propname ls_props[] = {
{ "DAV:", "getlastmodified" },
{ "DAV:", "getcontentlength" },
{ "DAV:", "resourcetype" },
{ "DAV:", "getetag"},
{ NULL, NULL }
};
@ -405,6 +412,120 @@ static int configureProxy( ne_session *session )
return re;
}
/*
* This hook is called for with the response of a request. Here its checked
* if a Set-Cookie header is there for the PHPSESSID. The key is stored into
* the webdav session to be added to subsequent requests.
*/
#define PHPSESSID "PHPSESSID="
static void post_request_hook(ne_request *req, void *userdata, const ne_status *status)
{
const char *set_cookie_header = NULL;
const char *sc = NULL;
char *key = NULL;
(void) userdata;
if(!(status && req)) return;
if( status->klass == 2 || status->code == 401 ) {
/* successful request */
set_cookie_header = ne_get_response_header( req, "Set-Cookie" );
if( set_cookie_header ) {
DEBUG_WEBDAV(" Set-Cookie found: %s", set_cookie_header);
/* try to find a ', ' sequence which is the separator of neon if multiple Set-Cookie
* headers are there.
* The following code parses a string like this:
* PHPSESSID=n8feu3dsbarpvvufqae9btn5fl7m7ikgh5ml1fg37v4i2cah7k41; path=/; HttpOnly */
sc = set_cookie_header;
while(sc) {
if( strlen(sc) > strlen(PHPSESSID) &&
strncmp( sc, PHPSESSID, strlen(PHPSESSID)) == 0 ) {
const char *sc_val = sc; /* + strlen(PHPSESSID); */
const char *sc_end = sc_val;
int cnt = 0;
int len = strlen(sc_val); /* The length of the rest of the header string. */
while( cnt < len && *sc_end != ';' && *sc_end != ',') {
cnt++;
sc_end++;
}
if( cnt == len ) {
/* exit: We are at the end. */
sc = NULL;
} else if( *sc_end == ';' ) {
/* We are at the end of the session key. */
int keylen = sc_end-sc_val;
if( key ) SAFE_FREE(key);
key = c_malloc(keylen+1);
strncpy( key, sc_val, keylen );
key[keylen] = '\0';
/* now search for a ',' to find a potential other header entry */
while(cnt < len && *sc_end != ',') {
cnt++;
sc_end++;
}
if( cnt < len )
sc = sc_end+2; /* mind the space after the comma */
else
sc = NULL;
} else if( *sc_end == ',' ) {
/* A new entry is to check. */
if( *(sc_end + 1) == ' ') {
sc = sc_end+2;
} else {
/* error condition */
sc = NULL;
}
}
} else {
/* It is not a PHPSESSID-Header but another one which we're not interested in.
* forward to next header entry (search ',' )
*/
int len = strlen(sc);
int cnt = 0;
while(cnt < len && *sc != ',') {
cnt++;
sc++;
}
if( cnt < len )
sc = sc+2; /* mind the space after the comma */
else
sc = NULL;
}
}
}
} else {
DEBUG_WEBDAV("Request failed, don't take session header.");
}
if( key ) {
DEBUG_WEBDAV("----> Session-key: %s", key);
SAFE_FREE(dav_session.session_key);
dav_session.session_key = key;
}
}
/*
* this hook is called just after a request has been created, before its sent.
* Here it is used to set the session cookie if available.
*/
static void request_created_hook(ne_request *req, void *userdata,
const char *method, const char *requri)
{
(void) userdata;
(void) method;
(void) requri;
if( !req ) return;
if(dav_session.session_key) {
/* DEBUG_WEBDAV("Setting PHPSESSID to %s", dav_session.session_key); */
ne_add_request_header(req, "Cookie", dav_session.session_key);
}
}
/*
* Connect to a DAV server
* This function sets the flag _connected if the connection is established
@ -490,6 +611,13 @@ static int dav_connect(const char *base_url) {
}
ne_redirect_register( dav_session.ctx );
/* Hook to get the Session ID */
ne_hook_post_headers( dav_session.ctx, post_request_hook, NULL );
/* Hook called when a request is built. It sets the PHPSESSID header */
ne_hook_create_request( dav_session.ctx, request_created_hook, NULL );
dav_session.session_key = NULL;
/* Proxy support */
proxystate = configureProxy( dav_session.ctx );
if( proxystate < 0 ) {
@ -522,6 +650,7 @@ static void results(void *userdata,
struct resource *newres = 0;
const char *clength, *modtime = NULL;
const char *resourcetype = NULL;
const char *md5sum = NULL;
const ne_status *status = NULL;
char *path = ne_path_unescape( uri->path );
@ -561,6 +690,7 @@ static void results(void *userdata,
modtime = ne_propset_value( set, &ls_props[0] );
clength = ne_propset_value( set, &ls_props[1] );
resourcetype = ne_propset_value( set, &ls_props[2] );
md5sum = ne_propset_value( set, &ls_props[3] );
newres->type = resr_normal;
if( clength == NULL && resourcetype && strncmp( resourcetype, "<DAV:collection>", 16 ) == 0) {
@ -579,6 +709,16 @@ static void results(void *userdata,
}
}
if( md5sum ) {
int len = strlen(md5sum)-2;
if( len > 0 ) {
/* Skip the " around the string coming back from the ne_propset_value call */
newres->md5 = c_malloc(len+1);
strncpy( newres->md5, md5sum+1, len );
newres->md5[len] = '\0';
}
}
/* prepend the new resource to the result list */
newres->next = fetchCtx->list;
fetchCtx->list = newres;
@ -686,11 +826,14 @@ static csync_vio_file_stat_t *resourceToFileStat( struct resource *res )
}
/* Correct the mtime of the file with the server time delta */
lfs->mtime = res->modtime;
lfs->mtime = res->modtime - dav_session.time_delta;
lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME;
lfs->size = res->size;
lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE;
if( res->md5 ) {
lfs->md5 = c_strdup(res->md5);
}
lfs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
return lfs;
}
@ -745,6 +888,7 @@ static void free_fetchCtx( struct listdir_context *ctx )
while( res ) {
SAFE_FREE(res->uri);
SAFE_FREE(res->name);
SAFE_FREE(res->md5);
newres = res->next;
SAFE_FREE(res);
@ -753,6 +897,22 @@ static void free_fetchCtx( struct listdir_context *ctx )
SAFE_FREE(ctx);
}
static void fill_stat_cache( csync_vio_file_stat_t *lfs ) {
if( _fs.name ) SAFE_FREE(_fs.name);
if( _fs.md5 ) SAFE_FREE(_fs.md5 );
if( !lfs) return;
_fs.name = c_strdup(lfs->name);
_fs.mtime = lfs->mtime;
_fs.fields = lfs->fields;
_fs.type = lfs->type;
_fs.size = lfs->size;
if( lfs->md5 ) {
_fs.md5 = c_strdup(lfs->md5);
}
}
/*
* file functions
@ -786,6 +946,7 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) {
* stat. If the cache matches, a http call is saved.
*/
if( _fs.name && strcmp( buf->name, _fs.name ) == 0 ) {
buf->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE;
@ -795,10 +956,16 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) {
buf->fields = _fs.fields;
buf->type = _fs.type;
buf->mtime = _fs.mtime;
buf->size = _fs.size;
buf->mode = _stat_perms( _fs.type );
buf->md5 = NULL;
if( _fs.md5 ) {
buf->md5 = c_strdup( _fs.md5 );
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
}
DEBUG_WEBDAV("stat results from fs cache - md5: %s", _fs.md5 ? _fs.md5 : "NULL");
} else {
DEBUG_WEBDAV("stat results fetched.");
/* fetch data via a propfind call. */
fetchCtx = c_malloc( sizeof( struct listdir_context ));
if( ! fetchCtx ) {
@ -809,7 +976,6 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) {
curi = _cleanPath( uri );
DEBUG_WEBDAV("I have no stat cache, call propfind for %s", curi );
fetchCtx->list = NULL;
fetchCtx->target = curi;
fetchCtx->include_target = 1;
@ -859,21 +1025,31 @@ static int owncloud_stat(const char *uri, csync_vio_file_stat_t *buf) {
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_SIZE;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_PERMISSIONS;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
buf->fields = lfs->fields;
buf->type = lfs->type;
buf->mtime = lfs->mtime;
buf->size = lfs->size;
buf->mode = _stat_perms( lfs->type );
buf->md5 = NULL;
if( lfs->md5 ) {
buf->md5 = c_strdup( lfs->md5 );
}
/* put the stat information to cache for subsequent calls */
fill_stat_cache( lfs );
/* fill the static stat buf as input for the stat function */
csync_vio_file_stat_destroy( lfs );
}
free_fetchCtx( fetchCtx );
}
DEBUG_WEBDAV("STAT result from propfind: %s, md5: %s", buf->name ? buf->name:"NULL",
buf->md5 ? buf->md5 : "NULL" );
}
DEBUG_WEBDAV("STAT result: %s, type=%d", buf->name ? buf->name:"NULL",
buf->type );
return 0;
}
@ -1004,9 +1180,9 @@ static char*_lastDir = NULL;
* bool time_sync_required - oC does not require the time sync
* int unix_extensions - oC supports unix extensions.
*/
static csync_vio_capabilities_t _owncloud_capabilities = { true, false, true, 1 };
static csync_vio_capabilities_t _owncloud_capabilities = { true, false, false, 1 };
static csync_vio_capabilities_t *owncloud_get_capabilities(void)
static csync_vio_capabilities_t *owncloud_capabilities(void)
{
#ifdef _WIN32
_owncloud_capabilities.unix_extensions = 0;
@ -1014,6 +1190,65 @@ static csync_vio_capabilities_t *owncloud_get_capabilities(void)
return &_owncloud_capabilities;
}
static const char* owncloud_file_id( const char *path )
{
ne_request *req = NULL;
const char *header = NULL;
char *uri = _cleanPath(path);
char *buf = NULL;
const char *cbuf = NULL;
csync_vio_file_stat_t *fs = NULL;
bool doHeadRequest= false; /* ownCloud server doesn't have good support for HEAD yet */
if( doHeadRequest ) {
/* Perform an HEAD request to the resource. HEAD delivers the
* ETag header back. */
req = ne_request_create(dav_session.ctx, "HEAD", uri);
ne_request_dispatch(req);
header = ne_get_response_header(req, "etag");
}
/* If the request went wrong or the server did not respond correctly
* (that can happen for collections) a stat call is done which translates
* into a PROPFIND request.
*/
if( ! header ) {
/* Clear the cache */
fill_stat_cache(NULL);
/* ... and do a stat call. */
fs = csync_vio_file_stat_new();
if(fs == NULL) {
DEBUG_WEBDAV( "owncloud_file_id: memory fault.");
errno = ENOMEM;
return NULL;
}
if( owncloud_stat( path, fs ) == 0 ) {
header = fs->md5;
}
}
/* In case the result is surrounded by "" cut them away. */
if( header ) {
if( header [0] == '"' && header[ strlen(header)-1] == '"') {
int len = strlen( header )-2;
buf = c_malloc( len+1 );
strncpy( buf, header+1, len );
buf[len] = '\0';
cbuf = buf;
/* do not free header here, as it belongs to the request */
} else {
cbuf = c_strdup(header);
}
}
DEBUG_WEBDAV("Get file ID for %s: %s", path, cbuf ? cbuf:"<null>");
if( fs ) csync_vio_file_stat_destroy(fs);
if( req ) ne_request_destroy(req);
SAFE_FREE(uri);
return cbuf;
}
static csync_vio_method_handle_t *owncloud_open(const char *durl,
int flags,
mode_t mode) {
@ -1031,6 +1266,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl,
const char *winTmpUtf8 = NULL;
csync_stat_t sb;
#endif
const char *etag_header = NULL;
struct transfer_context *writeCtx = NULL;
csync_vio_file_stat_t statBuf;
@ -1074,6 +1310,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl,
} else {
if( owncloud_stat( dir, &statBuf ) == 0 ) {
SAFE_FREE(statBuf.name);
SAFE_FREE(statBuf.md5);
DEBUG_WEBDAV("Directory of file to open exists.");
SAFE_FREE( _lastDir );
_lastDir = c_strdup(dir);
@ -1092,6 +1329,8 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl,
writeCtx = c_malloc( sizeof(struct transfer_context) );
writeCtx->bytes_written = 0;
writeCtx->clean_uri = c_strdup(uri);
if( rc == NE_OK ) {
/* open a temp file to store the incoming data */
#ifdef _WIN32
@ -1149,7 +1388,7 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl,
writeCtx->fileWritten = 0; /* flag to indicate if contents was pushed to file */
writeCtx->req = ne_request_create(dav_session.ctx, "PUT", uri);
writeCtx->method = "PUT";
writeCtx->method = "PUT";
}
@ -1197,6 +1436,13 @@ static csync_vio_method_handle_t *owncloud_open(const char *durl,
ne_decompress_destroy( writeCtx->decompress );
}
/* preserve the ETag header */
etag_header = ne_get_response_header( writeCtx->req, "ETag" );
if( etag_header ) {
writeCtx->md5 = c_strdup( etag_header );
DEBUG_WEBDAV("GET Etag: %s", etag_header );
}
/* delete the request in any case */
ne_request_destroy(writeCtx->req);
#else
@ -1243,7 +1489,7 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) {
int rc;
int ret = 0;
size_t len = 0;
const _TCHAR *tmpFileName = 0;
const _TCHAR *tmpFileName = NULL;
writeCtx = (struct transfer_context*) fhandle;
@ -1302,15 +1548,17 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) {
}
if (rc == NE_OK) {
if ( ne_get_status( writeCtx->req )->klass != 2 ) {
DEBUG_WEBDAV("Error - PUT status value no 2xx");
errno = EIO;
ret = -1;
// DEBUG_WEBDAV("Error - PUT status value no 2xx");
// errno = EIO;
// ret = -1;
}
} else {
DEBUG_WEBDAV("Error - put request on close failed: %d!", rc );
errno = EIO;
ret = -1;
}
/* Remove the local file. */
_tunlink(tmpFileName);
}
c_free_multibyte(tmpFileName);
} else {
@ -1320,9 +1568,9 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) {
rc = ne_request_dispatch( writeCtx->req );
if( rc == NE_OK ) {
if ( ne_get_status( writeCtx->req )->klass != 2 ) {
DEBUG_WEBDAV("Error - PUT status value no 2xx");
errno = EIO;
ret = -1;
// DEBUG_WEBDAV("Error - PUT status value no 2xx");
// errno = EIO;
// ret = -1;
}
} else {
DEBUG_WEBDAV("Error - put request from memory failed: %d!", rc );
@ -1331,6 +1579,7 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) {
}
}
}
ne_request_destroy( writeCtx->req );
} else {
/* Its a GET request, not much to do in close. */
@ -1341,11 +1590,10 @@ static int owncloud_close(csync_vio_method_handle_t *fhandle) {
}
}
}
/* Remove the local file. */
unlink( writeCtx->tmpFileName );
/* free mem. Note that the request mem is freed by the ne_request_destroy call */
SAFE_FREE( writeCtx->tmpFileName );
SAFE_FREE( writeCtx->clean_uri );
SAFE_FREE( writeCtx );
return ret;
@ -1481,11 +1729,7 @@ static csync_vio_file_stat_t *owncloud_readdir(csync_vio_method_handle_t *dhandl
fetchCtx->currResource = fetchCtx->currResource->next;
/* fill the static stat buf as input for the stat function */
_fs.name = lfs->name;
_fs.mtime = lfs->mtime;
_fs.fields = lfs->fields;
_fs.type = lfs->type;
_fs.size = lfs->size;
fill_stat_cache(lfs);
}
/* DEBUG_WEBDAV("LFS fields: %s: %d", lfs->name, lfs->type ); */
@ -1651,11 +1895,11 @@ static int owncloud_utimes(const char *uri, const struct timeval *times) {
pname.name = "lastmodified";
newmodtime = modtime->tv_sec;
#if TIMEDELTA
DEBUG_WEBDAV("Add a time delta to modtime %lu: %ld",
modtime->tv_sec, dav_session.time_delta);
newmodtime += dav_session.time_delta;
#endif
snprintf( val, sizeof(val), "%ld", newmodtime );
DEBUG_WEBDAV("Setting LastModified of %s to %s", curi, val );
@ -1678,7 +1922,8 @@ static int owncloud_utimes(const char *uri, const struct timeval *times) {
csync_vio_method_t _method = {
.method_table_size = sizeof(csync_vio_method_t),
.get_capabilities = owncloud_get_capabilities,
.get_capabilities = owncloud_capabilities,
.get_file_id = owncloud_file_id,
.open = owncloud_open,
.creat = owncloud_creat,
.close = owncloud_close,
@ -1744,6 +1989,9 @@ void vio_module_shutdown(csync_vio_method_t *method) {
SAFE_FREE( dav_session.proxy_user );
SAFE_FREE( dav_session.proxy_pwd );
/* free stat memory */
fill_stat_cache(NULL);
if( dav_session.ctx )
ne_session_destroy( dav_session.ctx );
/* DEBUG_WEBDAV( "********** vio_module_shutdown" ); */

8
src/CMakeLists.txt
View File

@ -4,11 +4,6 @@ add_subdirectory(std)
find_package(SQLite3 REQUIRED)
find_package(Iniparser REQUIRED)
if(CMAKE_CROSSCOMPILING)
find_package(OpenSSLCross REQUIRED)
else()
find_package(OpenSSL REQUIRED)
endif()
set(CSYNC_PUBLIC_INCLUDE_DIRS
${CMAKE_CURRENT_SOURCE_DIR}
@ -17,7 +12,6 @@ set(CSYNC_PUBLIC_INCLUDE_DIRS
)
set(CSYNC_PRIVATE_INCLUDE_DIRS
${OPENSSL_INCLUDE_DIRS}
${INIPARSER_INCLUDE_DIRS}
${SQLITE3_INCLUDE_DIRS}
${CSTDLIB_PUBLIC_INCLUDE_DIRS}
@ -34,7 +28,6 @@ set(CSYNC_LINK_LIBRARIES
${CSTDLIB_LIBRARY}
${CSYNC_REQUIRED_LIBRARIES}
${INIPARSER_LIBRARIES}
${OPENSSL_LIBRARIES}
${SQLITE3_LIBRARIES}
)
@ -48,6 +41,7 @@ set(csync_SRCS
csync_config.c
csync_exclude.c
csync_statedb.c
csync_dbtree.c
csync_time.c
csync_util.c
csync_misc.c

2
src/csync.c
View File

@ -733,8 +733,10 @@ int csync_destroy(CSYNC *ctx) {
/* free memory */
c_rbtree_free(ctx->local.tree);
c_list_free(ctx->local.list);
c_list_free(ctx->local.id_list);
c_rbtree_free(ctx->remote.tree);
c_list_free(ctx->remote.list);
c_list_free(ctx->remote.id_list);
SAFE_FREE(ctx->local.uri);
SAFE_FREE(ctx->remote.uri);
SAFE_FREE(ctx->options.config_dir);

2
src/csync.h
View File

@ -51,7 +51,7 @@ extern "C" {
/* csync version */
#define LIBCSYNC_VERSION_MAJOR 0
#define LIBCSYNC_VERSION_MINOR 50
#define LIBCSYNC_VERSION_MICRO 8
#define LIBCSYNC_VERSION_MICRO 10
#define LIBCSYNC_VERSION_INT CSYNC_VERSION_INT(LIBCSYNC_VERSION_MAJOR, \
LIBCSYNC_VERSION_MINOR, \

231
src/csync_dbtree.c Normal file
View File

@ -0,0 +1,231 @@
/*
* libcsync -- a library to sync a directory with another
*
* Copyright (c) 2012 by Klaas Freitag <freitag@owncloud.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "config.h"
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <sqlite3.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include "csync_dbtree.h"
#include "c_lib.h"
#include "csync_private.h"
#include "csync_statedb.h"
#include "csync_util.h"
#define CSYNC_LOG_CATEGORY_NAME "csync.dbtree"
#include "csync_log.h"
struct dir_listing {
c_list_t *list;
unsigned int cnt;
c_list_t *entry;
char *dir;
};
csync_vio_method_handle_t *csync_dbtree_opendir(CSYNC *ctx, const char *name)
{
char *stmt = NULL;
char *column = NULL;
const char *path = NULL;
csync_vio_file_stat_t *fs = NULL;
unsigned int c = 0;
c_strlist_t *list = NULL;
struct dir_listing *listing = NULL;
/* "phash INTEGER(8),"
"pathlen INTEGER,"
"path VARCHAR(4096),"
"inode INTEGER,"
"uid INTEGER,"
"gid INTEGER,"
"mode INTEGER,"
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32),"
*/
int col_count = 9;
if( strlen(name) < strlen(ctx->remote.uri)+1) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "name does not contain remote uri!");
return NULL;
}
path = name + strlen(ctx->remote.uri)+1;
if( asprintf( &stmt, "SELECT phash, path, inode, uid, gid, mode, modtime, type, md5 "
"FROM metadata WHERE path GLOB('%s/*')", path ) < 0 ) {
return NULL;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SQL: %s", stmt);
list = csync_statedb_query( ctx, stmt );
if( ! list ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Query result list is NULL!");
return NULL;
}
/* list count must be a multiple of col_count */
if( list->count % col_count != 0 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Wrong size of query result list");
return NULL;
}
listing = c_malloc(sizeof(struct dir_listing));
if( listing == NULL ) {
errno = ENOMEM;
return NULL;
}
listing->dir = c_strdup(path);
for( c = 0; c < (list->count / col_count); c++) {
int base = c*col_count;
int cnt = 0;
int tpath_len = 0;
int type = 0;
char *tpath = list->vector[base+1];
/* check if the result points to a file directly below the search path
* by checking if there is another / in the result.
* If yes, skip it.
* FIXME: Find a better filter solution here.
*/
tpath += strlen(path)+1; /* jump over the search path */
tpath_len = strlen( tpath );
while( cnt < tpath_len ) {
if(*(tpath+cnt) == '/') {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Skipping entry: %s", list->vector[base+1]);
break;
}
cnt++;
}
if( cnt < tpath_len ) continue;
fs = csync_vio_file_stat_new();
fs->fields = CSYNC_VIO_FILE_STAT_FIELDS_NONE;
column = list->vector[base+0]; /* phash */
column = list->vector[base+1]; /* path */
fs->name = c_strdup(column+strlen(path)+1);
column = list->vector[base+2]; /* inode */
fs->inode = atoi(column);
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_INODE;
column = list->vector[base+3]; /* uid */
fs->uid = atoi(column);
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_UID;
column = list->vector[base+4]; /* gid */
fs->gid = atoi(column);
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_GID;
column = list->vector[base+5]; /* mode */
fs->mode = atoi(column);
// fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_M;
column = list->vector[base+6]; /* modtime */
fs->mtime = strtoul(column, NULL, 10);
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MTIME;
column = list->vector[base+7]; /* type */
type = atoi(column);
/* Attention: the type of csync_ftw_type_e which is the source for
* the database entry is different from csync_vio_file_type_e which
* is the target file type here. Mapping is needed!
*/
switch( type ) {
case CSYNC_FTW_TYPE_DIR:
fs->type = CSYNC_VIO_FILE_TYPE_DIRECTORY;
break;
case CSYNC_FTW_TYPE_FILE:
fs->type = CSYNC_VIO_FILE_TYPE_REGULAR;
break;
case CSYNC_FTW_TYPE_SLINK:
fs->type = CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK;
break;
default:
fs->type = CSYNC_VIO_FILE_TYPE_UNKNOWN;
}
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE;
column = list->vector[base+8]; /* type */
fs->md5 = c_strdup(column);
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
/* store into result list. */
listing->list = c_list_append( listing->list, fs );
listing->cnt++;
}
listing->entry = c_list_first( listing->list );
c_strlist_destroy( list );
SAFE_FREE(stmt);
return listing;
}
int csync_dbtree_closedir(CSYNC *ctx, csync_vio_method_handle_t *dhandle)
{
struct dir_listing *dl = NULL;
int rc = 0;
(void) ctx;
if( dhandle != NULL ) {
dl = (struct dir_listing*) dhandle;
c_list_free(dl->list);
SAFE_FREE(dl->dir);
SAFE_FREE(dl);
}
return rc;
}
csync_vio_file_stat_t *csync_dbtree_readdir(CSYNC *ctx, csync_vio_method_handle_t *dhandle)
{
csync_vio_file_stat_t *fs = NULL;
struct dir_listing *dl = NULL;
(void) ctx;
if( dhandle != NULL ) {
dl = (struct dir_listing*) dhandle;
if( dl->entry != NULL ) {
fs = (csync_vio_file_stat_t*) dl->entry->data;
dl->entry = c_list_next( dl->entry);
}
}
return fs;
}
/* vim: set ts=8 sw=2 et cindent: */

60
src/csync_dbtree.h Normal file
View File

@ -0,0 +1,60 @@
/*
* libcsync -- a library to sync a directory with another
*
* Copyright (c) 2012 by Klaas Freitag <freitag@owncloud.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
/**
* @file csync_dbtree.h
*
* @brief Private interface of csync
*
* @defgroup csyncdbtreeInternals csync statedb internals
* @ingroup csyncInternalAPI
*
* @{
*/
#ifndef _CSYNC_DBTREE_H
#define _CSYNC_DBTREE_H
#include "c_lib.h"
#include "csync_private.h"
#include "vio/csync_vio_handle.h"
/**
* @brief Open a directory based on the statedb.
*
* This function reads the list of files within a directory from statedb and
* builds up a list in memory.
*
* @param ctx The csync context.
* @param name The directory name.
*
* @return 0 on success, less than 0 if an error occured with errno set.
*/
csync_vio_method_handle_t *csync_dbtree_opendir(CSYNC *ctx, const char *name);
int csync_dbtree_closedir(CSYNC *ctx, csync_vio_method_handle_t *dhandle);
csync_vio_file_stat_t *csync_dbtree_readdir(CSYNC *ctx, csync_vio_method_handle_t *dhandle);
/**
* }@
*/
#endif /* _CSYNC_DBTREE_H */
/* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */

6
src/csync_exclude.c
View File

@ -109,6 +109,7 @@ void csync_exclude_destroy(CSYNC *ctx) {
int csync_excluded(CSYNC *ctx, const char *path) {
size_t i;
const char *p;
const char *bname = NULL;
if (! ctx->options.unix_extensions) {
for (p = path; *p; p++) {
@ -133,11 +134,16 @@ int csync_excluded(CSYNC *ctx, const char *path) {
}
if (ctx->excludes->count) {
bname = c_basename(path);
for (i = 0; i < ctx->excludes->count; i++) {
if (csync_fnmatch(ctx->excludes->vector[i], path, 0) == 0) {
return 1;
}
if( bname && csync_fnmatch(ctx->excludes->vector[i], bname, 0) == 0) {
return 1;
}
}
SAFE_FREE(bname);
}
return 0;
}

1
src/csync_lock.c
View File

@ -63,6 +63,7 @@ static int _csync_lock_create(const char *lockfile) {
goto out;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Create temporary lock file: %s", ctmpfile);
if ((fd = mkstemp(ctmpfile)) < 0) {
strerror_r(errno, errbuf, sizeof(errbuf));

8
src/csync_private.h
View File

@ -94,6 +94,7 @@ struct csync_s {
char *uri;
c_rbtree_t *tree;
c_list_t *list;
c_list_t *id_list;
enum csync_replica_e type;
} local;
@ -101,7 +102,9 @@ struct csync_s {
char *uri;
c_rbtree_t *tree;
c_list_t *list;
c_list_t *id_list;
enum csync_replica_e type;
int read_from_db;
} remote;
struct {
@ -135,7 +138,7 @@ struct csync_s {
/* error code of the last operation */
enum csync_error_codes_e error_code;
int status;
};
@ -159,7 +162,10 @@ struct csync_file_stat_s {
mode_t mode; /* u32 */
int nlink; /* u32 */
int type; /* u32 */
char *destpath; /* for renames */
const char *md5;
enum csync_instructions_e instruction; /* u32 */
char path[1]; /* u8 */
}

263
src/csync_propagate.c
View File

@ -28,10 +28,12 @@
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include "csync_private.h"
#include "csync_propagate.h"
#include "vio/csync_vio.h"
#include "c_jhash.h"
#define CSYNC_LOG_CATEGORY_NAME "csync.propagator"
#include "csync_log.h"
@ -46,6 +48,27 @@ static int _csync_cleanup_cmp(const void *a, const void *b) {
return strcmp(st_a->path, st_b->path);
}
static void _store_id_update(CSYNC *ctx, csync_file_stat_t *st) {
c_list_t *list = NULL;
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SYNCED remember dir: %s", st->path);
switch (ctx->current) {
case LOCAL_REPLICA:
list = c_list_prepend(ctx->local.id_list, (void*)st);
if( list != NULL ) {
ctx->local.id_list = list;
}
break;
case REMOTE_REPLCIA:
list = c_list_prepend(ctx->remote.id_list, (void*)st);
if(list != NULL ) {
ctx->remote.id_list = list;
}
break;
}
}
static bool _push_to_tmp_first(CSYNC *ctx)
{
if( ctx->current == REMOTE_REPLCIA ) return true; /* Always push to tmp for destination local file system */
@ -56,6 +79,22 @@ static bool _push_to_tmp_first(CSYNC *ctx)
return false;
}
static const char*_get_md5( CSYNC *ctx, const char *path ) {
const char *md5 = NULL;
char *buf = NULL;
/* Always use the remote uri path, local does not have Ids. */
if (asprintf(&buf, "%s/%s", ctx->remote.uri, path) < 0) {
return 0;
}
md5 = csync_vio_file_id(ctx, buf);
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "MD5 for %s: %s", buf, md5 ? md5 : "<null>");
SAFE_FREE(buf);
return md5;
}
static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
enum csync_replica_e srep = -1;
enum csync_replica_e drep = -1;
@ -65,6 +104,7 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
char *duri = NULL;
char *turi = NULL;
char *tdir = NULL;
const char *tmd5 = NULL;
csync_vio_handle_t *sfp = NULL;
csync_vio_handle_t *dfp = NULL;
@ -332,6 +372,18 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
rc = 1;
goto out;
}
if( st->md5 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "UUUU MD5 sum: %s", st->md5);
} else {
if( tstat->md5 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Target MD5 sum is %s", tstat->md5 );
if(st->md5) SAFE_FREE(st->md5);
st->md5 = c_strdup(tstat->md5 );
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "MD5 sum is empty");
}
}
}
if (_push_to_tmp_first(ctx)) {
@ -386,6 +438,17 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
ctx->replica = drep;
csync_vio_utimes(ctx, duri, times);
/* For remote repos, after the utimes call, the ID has changed again */
/* do a stat on the target again to get a valid md5 */
tmd5 = _get_md5(ctx, st->path);
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "FINAL MD5: %s", tmd5 ? tmd5 : "<null>");
if(tmd5) {
SAFE_FREE(st->md5);
st->md5 = tmd5;
}
/* set instruction for the statedb merger */
st->instruction = CSYNC_INSTRUCTION_UPDATED;
@ -556,6 +619,8 @@ static int _csync_rename_file(CSYNC *ctx, csync_file_stat_t *st) {
struct timeval times[2];
char *suri = NULL;
char *duri = NULL;
const char *tmd5 = NULL;
c_rbnode_t *node = NULL;
switch (ctx->current) {
case REMOTE_REPLCIA:
@ -605,16 +670,36 @@ static int _csync_rename_file(CSYNC *ctx, csync_file_stat_t *st) {
csync_vio_utimes(ctx, duri, times);
/* set instruction for the statedb merger */
/* The the uniq ID for the destination */
tmd5 = _get_md5(ctx, st->destpath);
if( rc > -1 ) {
st->instruction = CSYNC_INSTRUCTION_RENAME;
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "RENAME file: %s", suri);
/* Find the destination entry in the local tree and insert the uniq id */
int len = strlen(st->destpath);
uint64_t h = c_jhash64((uint8_t *) st->destpath, len, 0);
h = c_jhash64((uint8_t *) st->destpath, len, 0);
/* search in the local tree for the local file to get the mtime */
node = c_rbtree_find(ctx->local.tree, &h);
if(node == NULL) {
/* no local file found. */
} else {
csync_file_stat_t *other = NULL;
/* set the mtime which is needed in statedb_get_uniqid */
other = (csync_file_stat_t *) node->data;
if( other ) {
other->md5 = tmd5;
}
}
/* set instruction for the statedb merger */
st->instruction = CSYNC_INSTRUCTION_DELETED;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "RENAME file: %s => %s with ID %s", st->path, st->destpath, st->md5);
out:
SAFE_FREE(suri);
SAFE_FREE(duri);
SAFE_FREE(st->destpath);
/* set instruction for the statedb merger */
if (rc != 0) {
@ -705,6 +790,7 @@ static int _csync_new_dir(CSYNC *ctx, csync_file_stat_t *st) {
enum csync_replica_e replica_bak;
char errbuf[256] = {0};
char *uri = NULL;
const char *tmd5 = NULL;
struct timeval times[2];
int rc = -1;
@ -775,6 +861,12 @@ static int _csync_new_dir(CSYNC *ctx, csync_file_stat_t *st) {
csync_vio_utimes(ctx, uri, times);
tmd5 = _get_md5(ctx, st->path);
if(tmd5) {
SAFE_FREE(st->md5);
st->md5 = tmd5;
}
/* set instruction for the statedb merger */
st->instruction = CSYNC_INSTRUCTION_UPDATED;
@ -798,6 +890,7 @@ static int _csync_sync_dir(CSYNC *ctx, csync_file_stat_t *st) {
enum csync_replica_e replica_bak;
char errbuf[256] = {0};
char *uri = NULL;
const char *tmd5 = NULL;
struct timeval times[2];
int rc = -1;
@ -851,11 +944,16 @@ static int _csync_sync_dir(CSYNC *ctx, csync_file_stat_t *st) {
times[0].tv_usec = times[1].tv_usec = 0;
csync_vio_utimes(ctx, uri, times);
tmd5 = _get_md5(ctx, st->path);
if(tmd5) {
SAFE_FREE(st->md5);
st->md5 = tmd5;
}
/* set instruction for the statedb merger */
st->instruction = CSYNC_INSTRUCTION_UPDATED;
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "SYNCED dir: %s", uri);
ctx->replica = replica_bak;
rc = 0;
@ -951,6 +1049,148 @@ out:
return rc;
}
static int _cmp_char( const void *d1, const void *d2 )
{
const char *c1 = (const char*) d1;
const char *c2 = (const char*) d2;
// CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "COMPARE: %s <-> %s", c1, c2);
if( c_streq(c1, c2) ) return 0;
return 1;
}
/*
* This function corrects the unique IDs of parent directories of changed
* files. Other than in the file system, the change of a unique Id propagates
* up to the top dir. To save the correct IDs, in all propagations, pathes
* are recorded in the local and remotes id_list lists.
* In this function, the unique ID is queried for each directory once and
* stored into the according entry.
*/
static int _csync_correct_id(CSYNC *ctx) {
c_list_t *walk = NULL;
c_list_t *seen_dirs = NULL;
c_list_t *list = NULL;
c_rbtree_t *tree = NULL;
const char *replica = NULL;
char *path = NULL;
switch (ctx->current) {
case LOCAL_REPLICA:
list = ctx->local.id_list;
tree = ctx->local.tree;
replica = "LOCAL_REPLICA";
break;
case REMOTE_REPLCIA:
list = ctx->remote.id_list;
tree = ctx->remote.tree;
replica = "REMOTE_REPLICA";
break;
default:
break;
}
if (list == NULL) {
return 0;
}
list = c_list_sort(list, _csync_cleanup_cmp);
if (list == NULL) {
return -1;
}
for (walk = c_list_last(list); walk != NULL; walk = c_list_prev(walk)) {
csync_file_stat_t *st = NULL;
st = (csync_file_stat_t *) walk->data;
if( st->type == CSYNC_FTW_TYPE_FILE ) {
path = c_dirname( st->path );
} else if( st->type == CSYNC_FTW_TYPE_DIR ) {
path = c_strdup(st->path); /* Allocate mem for this not to disappoint FREE */
} else {
/* unhandled path */
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "correct ID on dir: %s", path);
/* handle the . path */
if( path && path[0] == '.' && strlen(path) == 1) {
SAFE_FREE(path);
path = NULL;
}
while( path ) {
uint64_t h;
int len;
c_rbnode_t *node = NULL;
char pathbuf[PATH_MAX] = {'\0' };
char *old_path = path;
csync_file_stat_t *tfs = NULL;
/* do stuff with the dir here */
if( seen_dirs && c_list_find_custom( seen_dirs, path, _cmp_char)) {
// CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "saw this dir already: %s", path);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "climb on dir: %s (%s)", path, replica);
seen_dirs = c_list_prepend( seen_dirs, c_strdup(path));
/* Find the correct target entry. */
len = strlen(path);
h = c_jhash64((uint8_t *) path, len, 0);
node = c_rbtree_find(tree, &h);
if (node == NULL) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Unable to find node");
} else {
tfs = c_rbtree_node_data(node);
if( tfs ) {
if(tfs->instruction == CSYNC_INSTRUCTION_DELETED) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Skipping update of MD5 because item is deleted.");
} else {
if(tfs->md5) SAFE_FREE(tfs->md5);
tfs->md5 = _get_md5(ctx, path);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "MD5 for dir: %s %s (Instruction: %s)", tfs->path,
tfs->md5, csync_instruction_str(tfs->instruction));
if( tfs->md5 && tfs->instruction == CSYNC_INSTRUCTION_NONE ) {
/* set instruction for the statedb merger */
tfs->instruction = CSYNC_INSTRUCTION_UPDATED;
}
}
}
}
}
/* get the parent dir */
/* copy one byte less, omit the trailing slash */
strncpy( pathbuf, path, strlen(path)-1 );
/* and get the path name */
path = c_dirname( pathbuf );
/* free the old path memory */
SAFE_FREE(old_path );
/* exit on top directory */
if( c_streq(path, ".")) {
SAFE_FREE(path);
path = NULL;
}
}
}
/* Free the seen_dirs list */
if( seen_dirs ) {
c_list_t *walk1 = NULL;
for (walk1 = c_list_first(seen_dirs); walk1 != NULL; walk1 = c_list_next(walk1)) {
char *data = NULL;
data = (char*) walk1->data;
SAFE_FREE(data);
}
}
c_list_free(seen_dirs);
return 0;
}
static int _csync_propagation_cleanup(CSYNC *ctx) {
c_list_t *list = NULL;
c_list_t *walk = NULL;
@ -1020,30 +1260,35 @@ static int _csync_propagation_file_visitor(void *obj, void *data) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL NEW: %s",st->path);
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_RENAME:
if (_csync_rename_file(ctx, st) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL RENAME: %s",st->path);
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_SYNC:
if (_csync_sync_file(ctx, st) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL SYNC: %s",st->path);
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_REMOVE:
if (_csync_remove_file(ctx, st) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL REMOVE: %s",st->path);
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_CONFLICT:
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"case CSYNC_INSTRUCTION_CONFLICT: %s",st->path);
if (_csync_conflict_file(ctx, st) < 0) {
goto err;
}
_store_id_update(ctx, st);
break;
default:
break;
@ -1084,22 +1329,27 @@ static int _csync_propagation_dir_visitor(void *obj, void *data) {
if (_csync_new_dir(ctx, st) < 0) {
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_SYNC:
if (_csync_sync_dir(ctx, st) < 0) {
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_CONFLICT:
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"directory attributes different");
if (_csync_sync_dir(ctx, st) < 0) {
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_REMOVE:
if (_csync_remove_dir(ctx, st) < 0) {
goto err;
}
_store_id_update(ctx, st);
break;
case CSYNC_INSTRUCTION_RENAME:
/* there can't be a rename for dirs. See updater. */
break;
@ -1142,6 +1392,9 @@ int csync_propagate_files(CSYNC *ctx) {
return -1;
}
if( _csync_correct_id(ctx) < 0) {
return -1;
}
return 0;
}

493
src/csync_reconcile.c
View File

@ -43,269 +43,272 @@
* source and the destination, have been changed, the newer file wins.
*/
static int _csync_merge_algorithm_visitor(void *obj, void *data) {
csync_file_stat_t *cur = NULL;
csync_file_stat_t *other = NULL;
csync_file_stat_t *tmp = NULL;
uint64_t h = 0;
int len = 0;
csync_file_stat_t *cur = NULL;
csync_file_stat_t *other = NULL;
csync_file_stat_t *tmp = NULL;
uint64_t h = 0;
int len = 0;
CSYNC *ctx = NULL;
c_rbtree_t *tree = NULL;
c_rbnode_t *node = NULL;
CSYNC *ctx = NULL;
c_rbtree_t *tree = NULL;
c_rbnode_t *node = NULL;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
cur = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
/* we need the opposite tree! */
switch (ctx->current) {
/* we need the opposite tree! */
switch (ctx->current) {
case LOCAL_REPLICA:
tree = ctx->remote.tree;
break;
tree = ctx->remote.tree;
break;
case REMOTE_REPLCIA:
tree = ctx->local.tree;
break;
tree = ctx->local.tree;
break;
default:
break;
}
node = c_rbtree_find(tree, &cur->phash);
/* file only found on current replica */
if (node == NULL) {
switch(cur->instruction) {
/* file has been modified */
case CSYNC_INSTRUCTION_EVAL:
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
/* file has been removed on the opposite replica */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_REMOVE;
break;
case CSYNC_INSTRUCTION_RENAME:
/* rename support only on the local replica because of inode needed. */
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
/* Find the opposite node. */
if( tmp ) {
/* We need to calculate the phash again because of the phash being stored as int in db. */
if( tmp->path ) {
len = strlen( tmp->path );
h = c_jhash64((uint8_t *) tmp->path, len, 0);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"PHash of temporar opposite: %llu", h);
node = c_rbtree_find(tree, &h);
}
if(node) {
other = (csync_file_stat_t*)node->data;
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
if( ! other ) {
cur->instruction = CSYNC_INSTRUCTION_NEW;
}
}
}
break;
default:
break;
}
} else {
/*
node = c_rbtree_find(tree, &cur->phash);
/* file only found on current replica */
if (node == NULL) {
switch(cur->instruction) {
/* file has been modified */
case CSYNC_INSTRUCTION_EVAL:
cur->instruction = CSYNC_INSTRUCTION_NEW;
break;
/* file has been removed on the opposite replica */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_REMOVE;
break;
case CSYNC_INSTRUCTION_RENAME:
/* rename support only on the local replica because of inode needed. */
if(ctx->current == LOCAL_REPLICA ) {
/* use the old name to find the "other" node */
tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode);
/* Find the opposite node. */
if( tmp ) {
/* We need to calculate the phash again because of the phash being stored as int in db. */
if( tmp->path ) {
len = strlen( tmp->path );
h = c_jhash64((uint8_t *) tmp->path, len, 0);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"PHash of temporar opposite: %llu", h);
node = c_rbtree_find(tree, &h);
}
if(node) {
other = (csync_file_stat_t*)node->data;
other->instruction = CSYNC_INSTRUCTION_RENAME;
other->destpath = c_strdup( cur->path );
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
if( ! other ) {
cur->instruction = CSYNC_INSTRUCTION_NEW;
}
}
}
break;
default:
break;
}
} else {
/*
* file found on the other replica
*/
other = (csync_file_stat_t *) node->data;
other = (csync_file_stat_t *) node->data;
switch (cur->instruction) {
/* file on current replica is new */
case CSYNC_INSTRUCTION_NEW:
switch (other->instruction) {
/* file on other replica is new too */
case CSYNC_INSTRUCTION_NEW:
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
other->instruction = CSYNC_INSTRUCTION_NONE;
}
} else if (cur->modtime < other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file are equal */
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_NONE;
switch (cur->instruction) {
/* file on current replica is new */
case CSYNC_INSTRUCTION_NEW:
switch (other->instruction) {
/* file on other replica is new too */
case CSYNC_INSTRUCTION_NEW:
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
other->instruction = CSYNC_INSTRUCTION_NONE;
}
} else if (cur->modtime < other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"file new on both, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file are equal */
/* FIXME: Get the id from the server! */
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction = CSYNC_INSTRUCTION_NONE;
if( !cur->md5 ) cur->md5 = other->md5;
}
break;
/* file on other replica has changed too */
case CSYNC_INSTRUCTION_EVAL:
/* file on current replica is newer */
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file on opposite replica is newer */
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
default:
break;
}
break;
/* file on other replica has changed too */
case CSYNC_INSTRUCTION_EVAL:
/* file on current replica is newer */
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file on opposite replica is newer */
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"new on cur, modified on other, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
/* file on current replica has been modified */
case CSYNC_INSTRUCTION_EVAL:
switch (other->instruction) {
/* file on other replica is new too */
case CSYNC_INSTRUCTION_NEW:
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
}
break;
/* file on other replica has changed too */
case CSYNC_INSTRUCTION_EVAL:
/* file on current replica is newer */
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction= CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file on opposite replica is newer */
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction=CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
default:
break;
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
default:
default:
break;
}
break;
/* file on current replica has been modified */
case CSYNC_INSTRUCTION_EVAL:
switch (other->instruction) {
/* file on other replica is new too */
case CSYNC_INSTRUCTION_NEW:
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"modified on cur, new on other, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
}
break;
/* file on other replica has changed too */
case CSYNC_INSTRUCTION_EVAL:
/* file on current replica is newer */
if (cur->modtime > other->modtime) {
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, cur is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_CONFLICT;
other->instruction= CSYNC_INSTRUCTION_NONE;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_SYNC;
}
} else {
/* file on opposite replica is newer */
if(ctx->options.with_conflict_copys)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"both modified, other is newer PATH=./%s",cur->path);
cur->instruction = CSYNC_INSTRUCTION_NONE;
other->instruction=CSYNC_INSTRUCTION_CONFLICT;
}
else
{
cur->instruction = CSYNC_INSTRUCTION_NONE;
}
}
break;
/* file on the other replica has not been modified */
case CSYNC_INSTRUCTION_NONE:
cur->instruction = CSYNC_INSTRUCTION_SYNC;
break;
default:
break;
}
break;
default:
break;
}
}
//hide instruction NONE messages when log level is set to debug,
//only show these messages on log level trace
if(cur->instruction ==CSYNC_INSTRUCTION_NONE)
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
else
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
return 0;
//hide instruction NONE messages when log level is set to debug,
//only show these messages on log level trace
if(cur->instruction ==CSYNC_INSTRUCTION_NONE)
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
else
{
if(cur->type == CSYNC_FTW_TYPE_DIR)
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s dir: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
else
{
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
"%-20s file: %s",
csync_instruction_str(cur->instruction),
cur->path);
}
}
return 0;
}
int csync_reconcile_updates(CSYNC *ctx) {

115
src/csync_statedb.c
View File

@ -213,6 +213,8 @@ int csync_statedb_create_tables(CSYNC *ctx) {
"gid INTEGER,"
"mode INTEGER,"
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32),"
"PRIMARY KEY(phash)"
");"
);
@ -232,6 +234,8 @@ int csync_statedb_create_tables(CSYNC *ctx) {
"gid INTEGER,"
"mode INTEGER,"
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32),"
"PRIMARY KEY(phash)"
");"
);
@ -254,6 +258,13 @@ int csync_statedb_create_tables(CSYNC *ctx) {
}
c_strlist_destroy(result);
result = csync_statedb_query(ctx,
"CREATE INDEX metadata_md5 ON metadata(md5);");
if (result == NULL) {
return -1;
}
c_strlist_destroy(result);
return 0;
}
@ -296,8 +307,8 @@ static int _insert_metadata_visitor(void *obj, void *data) {
case CSYNC_INSTRUCTION_CONFLICT:
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"SQL statement: INSERT INTO metadata_temp \n"
"\t\t\t(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES \n"
"\t\t\t(%llu, %lu, %s, %llu, %u, %u, %u, %lu);",
"\t\t\t(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES \n"
"\t\t\t(%llu, %lu, %s, %llu, %u, %u, %u, %lu, %d, %s);",
(long long unsigned int) fs->phash,
(long unsigned int) fs->pathlen,
fs->path,
@ -305,14 +316,16 @@ static int _insert_metadata_visitor(void *obj, void *data) {
fs->uid,
fs->gid,
fs->mode,
fs->modtime);
fs->modtime,
fs->type,
fs->md5 ? fs->md5 : "<empty>");
/*
* The phash needs to be long long unsigned int or it segfaults on PPC
*/
stmt = sqlite3_mprintf("INSERT INTO metadata_temp "
"(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES "
"(%llu, %lu, '%q', %llu, %u, %u, %u, %lu);",
"(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES "
"(%llu, %lu, '%q', %llu, %u, %u, %u, %lu, %d, '%s');",
(long long unsigned int) fs->phash,
(long unsigned int) fs->pathlen,
fs->path,
@ -320,7 +333,9 @@ static int _insert_metadata_visitor(void *obj, void *data) {
fs->uid,
fs->gid,
fs->mode,
fs->modtime);
fs->modtime,
fs->type,
fs->md5);
if (stmt == NULL) {
return -1;
@ -381,19 +396,19 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash) {
return NULL;
}
if (result->count <= 6) {
c_strlist_destroy(result);
return NULL;
}
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
len = strlen(result->vector[2]);
st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (st == NULL) {
c_strlist_destroy(result);
return NULL;
if (result->count != 0 && result->count < 10) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Amount of result columns wrong, db version mismatch!");
}
if(result->count > 7) {
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
len = strlen(result->vector[2]);
st = c_malloc(sizeof(csync_file_stat_t) + len + 1);
if (st == NULL) {
c_strlist_destroy(result);
return NULL;
}
/*
/*
* FIXME:
* We use an INTEGER(8) which is signed to the phash in the sqlite3 db,
* but the phash is an uint64_t. So for some values we get a string like
@ -403,17 +418,28 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash) {
* st->phash = strtoull(result->vector[0], NULL, 10);
*/
/* The query suceeded so use the phash we pass to the function. */
st->phash = phash;
/* The query suceeded so use the phash we pass to the function. */
st->phash = phash;
st->pathlen = atoi(result->vector[1]);
memcpy(st->path, (len ? result->vector[2] : ""), len + 1);
st->inode = atoi(result->vector[3]);
st->uid = atoi(result->vector[4]);
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
st->pathlen = atoi(result->vector[1]);
memcpy(st->path, (len ? result->vector[2] : ""), len + 1);
st->inode = atoi(result->vector[3]);
st->uid = atoi(result->vector[4]);
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "No result record found for phash = %llu",
(long long unsigned int) phash);
}
if(result->count > 8 && result->vector[8]) {
st->type = atoi(result->vector[8]);
}
if(result->count > 9 && result->vector[9]) {
st->md5 = c_strdup( result->vector[9] );
}
c_strlist_destroy(result);
return st;
@ -458,12 +484,38 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode) {
st->gid = atoi(result->vector[5]);
st->mode = atoi(result->vector[6]);
st->modtime = strtoul(result->vector[7], NULL, 10);
st->type = atoi(result->vector[8]);
if( result->vector[9] )
st->md5 = c_strdup(result->vector[9]);
c_strlist_destroy(result);
return st;
}
char *csync_statedb_get_uniqId( CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf ) {
char *ret = NULL;
c_strlist_t *result = NULL;
char *stmt = NULL;
stmt = sqlite3_mprintf("SELECT md5 FROM metadata WHERE phash='%llu' AND modtime=%lu", jHash, buf->mtime);
result = csync_statedb_query(ctx, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
}
if (result->count == 1) {
/* phash, pathlen, path, inode, uid, gid, mode, modtime */
ret = c_strdup( result->vector[0] );
}
c_strlist_destroy(result);
return ret;
}
/* query the statedb, caller must free the memory */
c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) {
int err = SQLITE_OK;
@ -475,6 +527,7 @@ c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) {
sqlite3_stmt *stmt;
const char *tail = NULL;
c_strlist_t *result = NULL;
int row = 0;
do {
/* compile SQL program into a virtual machine, reattempteing if busy */
@ -527,14 +580,20 @@ c_strlist_t *csync_statedb_query(CSYNC *ctx, const char *statement) {
break;
}
result = c_strlist_new(column_count);
row++;
if( result ) {
result = c_strlist_expand(result, row*column_count);
} else {
result = c_strlist_new(column_count);
}
if (result == NULL) {
return NULL;
}
/* iterate over columns */
for (i = 0; i < column_count; i++) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_column_text: %s", (char *) sqlite3_column_text(stmt, i));
// CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_column_text: %s", (char *) sqlite3_column_text(stmt, i));
if (c_strlist_add(result, (char *) sqlite3_column_text(stmt, i)) < 0) {
c_strlist_destroy(result);
return NULL;

1
src/csync_statedb.h
View File

@ -61,6 +61,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx, uint64_t phash);
csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode);
char *csync_statedb_get_uniqId( CSYNC *ctx, uint64_t jHash, csync_vio_file_stat_t *buf );
/**
* @brief A generic statedb query.
*

186
src/csync_update.c
View File

@ -103,37 +103,56 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
*/
if (csync_get_statedb_exists(ctx)) {
tmp = csync_statedb_get_stat_by_hash(ctx, h);
if (tmp && tmp->phash == h) {
/* we have an update! */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "time compare: %lu <-> %lu",
fs->mtime, tmp->modtime);
if (fs->mtime > tmp->modtime) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
} else {
/* check if it's a file and has been renamed */
if (type == CSYNC_FTW_TYPE_FILE && ctx->current == LOCAL_REPLICA) {
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
if (tmp && tmp->inode == fs->inode) {
/* inode found so the file has been renamed */
st->instruction = CSYNC_INSTRUCTION_RENAME;
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
#if 0
/* this code could possibly replace the one in csync_vio.c stat and would be more efficient */
if(tmp) {
if( ctx->current == LOCAL_REPLICA ) {
if(fs->mtime == tmp->modtime && fs->size == tmp->size) {
/* filesystem modtime is still the same as the db mtime
* thus the md5 sum is still valid. */
fs->md5 = c_strdup( tmp->md5 );
}
}
}
/* directory, remote and file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
}
#endif
if(tmp && tmp->phash == h ) { /* there is an entry in the database */
/* we have an update! */
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "time compare: %lu <-> %lu, md5: %s <-> %s",
fs->mtime, tmp->modtime, fs->md5, tmp->md5);
if( !fs->md5) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
if( !c_streq(fs->md5, tmp->md5 )) {
// if (!fs->mtime > tmp->modtime) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
st->instruction = CSYNC_INSTRUCTION_NONE;
} else {
/* check if it's a file and has been renamed */
if (type == CSYNC_FTW_TYPE_FILE && ctx->current == LOCAL_REPLICA) {
tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
if (tmp && tmp->inode == fs->inode) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "inodes: %ld <-> %ld", tmp->inode, fs->inode);
/* inode found so the file has been renamed */
st->instruction = CSYNC_INSTRUCTION_RENAME;
goto out;
} else {
/* file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
goto out;
}
}
/* directory, remote and file not found in statedb */
st->instruction = CSYNC_INSTRUCTION_NEW;
}
} else {
st->instruction = CSYNC_INSTRUCTION_NEW;
st->instruction = CSYNC_INSTRUCTION_NEW;
}
out:
if( tmp) SAFE_FREE(tmp->md5);
SAFE_FREE(tmp);
st->inode = fs->inode;
st->mode = fs->mode;
@ -143,7 +162,10 @@ out:
st->gid = fs->gid;
st->nlink = fs->nlink;
st->type = type;
st->md5 = NULL;
if( fs->md5 ) {
st->md5 = c_strdup(fs->md5);
}
st->phash = h;
st->pathlen = len;
memcpy(st->path, (len ? path : ""), len + 1);
@ -204,6 +226,66 @@ int csync_walker(CSYNC *ctx, const char *file, const csync_vio_file_stat_t *fs,
return 0;
}
/* check if the dirent entries for the directory can be read from db
* instead really calling readdir which is costly over net.
* For that, a single HEAD request is done on the directory to get its
* id. If the ID has not changed remotely, this subtree hasn't changed
* and can be read from db.
*/
static int _check_read_from_db(CSYNC *ctx, const char *uri) {
int len;
uint64_t h;
csync_vio_file_stat_t *fs = NULL;
char *md5_local = NULL;
const char *md5_remote = NULL;
const char *mpath;
c_rbnode_t *node = NULL;
int rc = 0; /* FIXME: Error handling! */
if( !c_streq( ctx->remote.uri, uri )) {
/* FIXME: The top uri can not be checked because there is no db entry for it */
if( strlen(uri) < strlen(ctx->remote.uri)+1 ) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "check_read_from_db: uri is not a remote uri.");
/* FIXME: errno? */
return -1;
}
mpath = uri + strlen(ctx->remote.uri) + 1;
fs = csync_vio_file_stat_new();
if(fs == NULL) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "check_read_from_db: memory fault.");
errno = ENOMEM;
return -1;
}
len = strlen( mpath );
h = c_jhash64((uint8_t *) mpath, len, 0);
/* search in the local tree for the local file to get the mtime */
node = c_rbtree_find(ctx->local.tree, &h);
if(node == NULL) {
/* no local file found. */
} else {
csync_file_stat_t *other = NULL;
/* set the mtime which is needed in statedb_get_uniqid */
other = (csync_file_stat_t *) node->data;
if( other )
fs->mtime = other->modtime;
}
md5_local = csync_statedb_get_uniqId( ctx, h, fs );
md5_remote = csync_vio_file_id(ctx, uri);
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Compare directory ids for %s: %s -> %s", mpath, md5_local, md5_remote );
if( c_streq(md5_local, md5_remote) ) {
ctx->remote.read_from_db = 1;
}
SAFE_FREE(md5_local);
SAFE_FREE(md5_remote);
csync_vio_file_stat_destroy(fs);
}
return rc;
}
/* File tree walker */
int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
unsigned int depth) {
@ -213,13 +295,30 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
csync_vio_handle_t *dh = NULL;
csync_vio_file_stat_t *dirent = NULL;
csync_vio_file_stat_t *fs = NULL;
int read_from_db = 0;
int rc = 0;
int res = 0;
bool do_read_from_db = (ctx->current == REMOTE_REPLCIA && ctx->remote.read_from_db);
if (uri[0] == '\0') {
errno = ENOENT;
goto error;
}
/* If remote, compare the id with the local id. If equal, read all contents from
* the database. */
read_from_db = ctx->remote.read_from_db;
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Incoming read_from_db-Flag for %s: %d",
uri, read_from_db );
if( ctx->current == REMOTE_REPLCIA && !do_read_from_db ) {
_check_read_from_db(ctx, uri);
do_read_from_db = (ctx->current == REMOTE_REPLCIA && ctx->remote.read_from_db);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Checking for read from db for %s: %d",
uri, ctx->remote.read_from_db );
}
if ((dh = csync_vio_opendir(ctx, uri)) == NULL) {
/* permission denied */
if (errno == EACCES) {
@ -280,8 +379,16 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
continue;
}
fs = csync_vio_file_stat_new();
if (csync_vio_stat(ctx, filename, fs) == 0) {
/* == see if really stat has to be called. */
if( do_read_from_db ) {
fs = dirent;
res = 0;
} else {
fs = csync_vio_file_stat_new();
res = csync_vio_stat(ctx, filename, fs);
}
if( res == 0) {
switch (fs->type) {
case CSYNC_VIO_FILE_TYPE_SYMBOLIC_LINK:
flag = CSYNC_FTW_FLAG_SLINK;
@ -305,11 +412,29 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
flag = CSYNC_FTW_FLAG_NSTAT;
}
if( ctx->current == LOCAL_REPLICA ) {
char *md5 = NULL;
int len = strlen( path );
uint64_t h = c_jhash64((uint8_t *) path, len, 0);
md5 = csync_statedb_get_uniqId( ctx, h, fs );
if( md5 ) {
SAFE_FREE(fs->md5);
fs->md5 = md5;
fs->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Uniq ID read from Database: %s -> %s", path, fs->md5 ? fs->md5 : "<NULL>" );
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "walk: %s", filename);
/* Call walker function for each file */
rc = fn(ctx, filename, fs, flag);
csync_vio_file_stat_destroy(fs);
if( ! do_read_from_db )
csync_vio_file_stat_destroy(fs);
else
SAFE_FREE(fs->md5);
if (rc < 0) {
csync_vio_closedir(ctx, dh);
@ -328,6 +453,9 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
dirent = NULL;
}
csync_vio_closedir(ctx, dh);
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Closing walk for %s with read_from_db %d", uri, read_from_db);
ctx->remote.read_from_db = read_from_db;
done:
csync_vio_file_stat_destroy(dirent);

126
src/csync_util.c
View File

@ -32,14 +32,6 @@
#include "csync_util.h"
#include "vio/csync_vio.h"
#if defined(__APPLE__)
# define COMMON_DIGEST_FOR_OPENSSL
# include <CommonCrypto/CommonDigest.h>
# define SHA1 CC_SHA1
#else
# include <openssl/md5.h>
#endif
#define CSYNC_LOG_CATEGORY_NAME "csync.util"
#include "csync_log.h"
@ -113,6 +105,7 @@ void csync_memstat_check(void) {
static int _merge_file_trees_visitor(void *obj, void *data) {
csync_file_stat_t *fs = NULL;
csync_file_stat_t *tfs = NULL;
csync_vio_file_stat_t *vst = NULL;
CSYNC *ctx = NULL;
@ -181,25 +174,31 @@ static int _merge_file_trees_visitor(void *obj, void *data) {
fs = c_rbtree_node_data(node);
switch (ctx->current) {
case LOCAL_REPLICA:
if (asprintf(&uri, "%s/%s", ctx->local.uri, fs->path) < 0) {
rc = -1;
strerror_r(errno, errbuf, sizeof(errbuf));
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s",
errbuf);
goto out;
case LOCAL_REPLICA:
/* If there is a destpath, this is a rename and the target must be used. */
if( fs->destpath ) {
asprintf(&uri, "%s/%s", ctx->local.uri, fs->destpath);
SAFE_FREE(fs->destpath);
} else {
if (asprintf(&uri, "%s/%s", ctx->local.uri, fs->path) < 0) {
rc = -1;
strerror_r(errno, errbuf, sizeof(errbuf));
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s",
errbuf);
goto out;
}
}
break;
case REMOTE_REPLCIA:
case REMOTE_REPLCIA:
if (asprintf(&uri, "%s/%s", ctx->remote.uri, fs->path) < 0) {
rc = -1;
strerror_r(errno, errbuf, sizeof(errbuf));
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s",
errbuf);
goto out;
rc = -1;
strerror_r(errno, errbuf, sizeof(errbuf));
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "file uri alloc failed: %s",
errbuf);
goto out;
}
break;
default:
default:
break;
}
@ -219,7 +218,24 @@ static int _merge_file_trees_visitor(void *obj, void *data) {
fs->inode = vst->inode;
fs->modtime = vst->mtime;
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: UPDATED", uri);
/* update with the id from the remote repo. This method always works on the local repo */
node = c_rbtree_find(ctx->remote.tree, &fs->phash);
if (node == NULL) {
rc = -1;
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Unable to find node");
goto out;
}
tfs = c_rbtree_node_data(node);
if( tfs && tfs->md5 ) {
if( fs->md5 ) SAFE_FREE(fs->md5);
fs->md5 = c_strdup( tfs->md5 );
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "PRE UPDATED %s: %s", fs->path, fs->md5);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "md5 is empty in merger!");
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "file: %s, instruction: UPDATED (%s)", uri, fs->md5);
fs->instruction = CSYNC_INSTRUCTION_NONE;
@ -334,68 +350,4 @@ uint64_t csync_create_statedb_hash(CSYNC *ctx) {
return hash;
}
static char* digest_to_out(unsigned char* digest)
{
char *out = c_malloc(33);
int n;
if( !digest ) return NULL;
for (n = 0; n < 16; ++n) {
snprintf(&(out[n*2]), 16*2, "%02x", (unsigned int) *(digest+n));
}
return out;
}
#define BUF_SIZE 512
char* csync_file_md5(const char *filename)
{
const char *tmpFileName;
int fd;
MD5_CTX c;
char buf[ BUF_SIZE+1 ];
unsigned char digest[16];
size_t size;
tmpFileName = c_multibyte( filename );
if ( (fd = _topen( tmpFileName, O_RDONLY )) < 0) {
return NULL;
} else {
MD5_Init(&c);
while( (size=read(fd, buf, BUF_SIZE )) > 0) {
buf[size]='\0';
MD5_Update(&c, buf, size);
}
close(fd);
MD5_Final(digest, &c);
}
c_free_multibyte(tmpFileName);
return digest_to_out(digest);
}
char* csync_buffer_md5(const char *str, int length)
{
MD5_CTX c;
unsigned char digest[16];
MD5_Init(&c);
while (length > 0) {
if (length > 512) {
MD5_Update(&c, str, 512);
} else {
MD5_Update(&c, str, length);
}
length -= 512;
str += 512;
}
MD5_Final(digest, &c);
return digest_to_out(digest);
}
/* vim: set ts=8 sw=2 et cindent: */

8
src/csync_util.h
View File

@ -36,13 +36,5 @@ int csync_unix_extensions(CSYNC *ctx);
/* Normalize the uri to <host>/<path> */
uint64_t csync_create_statedb_hash(CSYNC *ctx);
/* Calculate the md5 sum for a file given by filename.
* Caller has to free the memory. */
char* csync_file_md5(const char *filename);
/* Create an md5 sum from a data pointer with a given length.
* Caller has to free the memory */
char* csync_buffer_md5(const char *str, int length);
#endif /* _CSYNC_UTIL_H */
/* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */

39
src/vio/csync_vio.c
View File

@ -29,9 +29,12 @@
#include <dlfcn.h> /* dlopen(), dlclose(), dlsym() ... */
#include "csync_private.h"
#include "csync_dbtree.h"
#include "vio/csync_vio.h"
#include "vio/csync_vio_handle_private.h"
#include "vio/csync_vio_local.h"
#include "csync_statedb.h"
#include "std/c_jhash.h"
#define CSYNC_LOG_CATEGORY_NAME "csync.vio.main"
@ -193,6 +196,11 @@ int csync_vio_init(CSYNC *ctx, const char *module, const char *args) {
return -1;
}
if (! VIO_METHOD_HAS_FUNC(m, get_file_id)) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "module %s has no get_file_id fn", module);
}
ctx->module.method = m;
return 0;
@ -353,7 +361,12 @@ csync_vio_handle_t *csync_vio_opendir(CSYNC *ctx, const char *name) {
switch(ctx->replica) {
case REMOTE_REPLCIA:
mh = ctx->module.method->opendir(name);
if(ctx->remote.read_from_db) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Reading directory %s from database", name);
mh = csync_dbtree_opendir(ctx, name);
} else {
mh = ctx->module.method->opendir(name);
}
break;
case LOCAL_REPLICA:
mh = csync_vio_local_opendir(name);
@ -380,7 +393,11 @@ int csync_vio_closedir(CSYNC *ctx, csync_vio_handle_t *dhandle) {
switch(ctx->replica) {
case REMOTE_REPLCIA:
rc = ctx->module.method->closedir(dhandle->method_handle);
if(ctx->remote.read_from_db) {
rc = csync_dbtree_closedir(ctx, dhandle->method_handle);
} else {
rc = ctx->module.method->closedir(dhandle->method_handle);
}
break;
case LOCAL_REPLICA:
rc = csync_vio_local_closedir(dhandle->method_handle);
@ -400,7 +417,11 @@ csync_vio_file_stat_t *csync_vio_readdir(CSYNC *ctx, csync_vio_handle_t *dhandle
switch(ctx->replica) {
case REMOTE_REPLCIA:
fs = ctx->module.method->readdir(dhandle->method_handle);
if(ctx->remote.read_from_db) {
fs = csync_dbtree_readdir(ctx, dhandle->method_handle);
} else {
fs = ctx->module.method->readdir(dhandle->method_handle);
}
break;
case LOCAL_REPLICA:
fs = csync_vio_local_readdir(dhandle->method_handle);
@ -508,6 +529,17 @@ int csync_vio_rmdir(CSYNC *ctx, const char *uri) {
return rc;
}
const char *csync_vio_file_id(CSYNC *ctx, const char *path)
{
const char *re = NULL;
/* We always use the remote method here. */
if(ctx->module.method &&
VIO_METHOD_HAS_FUNC(ctx->module.method, get_file_id)) {
re = ctx->module.method->get_file_id(path);
}
return re;
}
int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) {
int rc = -1;
@ -520,7 +552,6 @@ int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf) {
#ifdef _WIN32
CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Win32: STAT-inode for %s: %llu", uri, buf->inode );
#endif
break;
default:
break;

1
src/vio/csync_vio.h
View File

@ -48,6 +48,7 @@ int csync_vio_mkdir(CSYNC *ctx, const char *uri, mode_t mode);
int csync_vio_mkdirs(CSYNC *ctx, const char *uri, mode_t mode);
int csync_vio_rmdir(CSYNC *ctx, const char *uri);
const char *csync_vio_file_id(CSYNC *ctx, const char *path);
int csync_vio_stat(CSYNC *ctx, const char *uri, csync_vio_file_stat_t *buf);
int csync_vio_rename(CSYNC *ctx, const char *olduri, const char *newuri);
int csync_vio_unlink(CSYNC *ctx, const char *uri);

10
src/vio/csync_vio_file_stat.c
View File

@ -30,7 +30,7 @@ csync_vio_file_stat_t *csync_vio_file_stat_new(void) {
if (file_stat == NULL) {
return NULL;
}
file_stat->md5 = NULL;
return file_stat;
}
@ -40,13 +40,15 @@ void csync_vio_file_stat_destroy(csync_vio_file_stat_t *file_stat) {
return;
}
if (file_stat->fields == CSYNC_VIO_FILE_STAT_FIELDS_SYMLINK_NAME) {
if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_SYMLINK_NAME) {
SAFE_FREE(file_stat->u.symlink_name);
}
if (file_stat->fields == CSYNC_VIO_FILE_STAT_FIELDS_CHECKSUM) {
if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_CHECKSUM) {
SAFE_FREE(file_stat->u.checksum);
}
if (file_stat->fields & CSYNC_VIO_FILE_STAT_FIELDS_MD5) {
SAFE_FREE(file_stat->md5);
}
SAFE_FREE(file_stat->name);
SAFE_FREE(file_stat);
}

2
src/vio/csync_vio_file_stat.h
View File

@ -72,6 +72,7 @@ enum csync_vio_file_stat_fields_e {
CSYNC_VIO_FILE_STAT_FIELDS_ACL = 1 << 14,
CSYNC_VIO_FILE_STAT_FIELDS_UID = 1 << 15,
CSYNC_VIO_FILE_STAT_FIELDS_GID = 1 << 16,
CSYNC_VIO_FILE_STAT_FIELDS_MD5 = 1 << 17,
};
@ -83,6 +84,7 @@ struct csync_vio_file_stat_s {
void *acl;
char *name;
char *md5;
uid_t uid;
gid_t gid;

7
src/vio/csync_vio_local.c
View File

@ -35,6 +35,8 @@
#include "c_private.h"
#include "c_lib.h"
#include "c_string.h"
#include "csync_util.h"
#include "csync_log.h"
#include "vio/csync_vio_local.h"
@ -387,6 +389,11 @@ int csync_vio_local_stat(const char *uri, csync_vio_file_stat_t *buf) {
buf->ctime = sb.st_ctime;
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_CTIME;
/* buf->md5 = csync_file_md5( uri ); */
/* md5 is queried one level higher because we do not have ctx here. */
buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_MD5;
c_free_multibyte(wuri);
return 0;
}

3
src/vio/csync_vio_method.h
View File

@ -52,7 +52,7 @@ typedef csync_vio_method_t *(*csync_vio_method_init_fn)(const char *method_name,
typedef void (*csync_vio_method_finish_fn)(csync_vio_method_t *method);
typedef csync_vio_capabilities_t *(*csync_method_get_capabilities_fn)(void);
typedef const char* (*csync_method_get_file_id_fn)(const char* path);
typedef csync_vio_method_handle_t *(*csync_method_open_fn)(const char *durl, int flags, mode_t mode);
typedef csync_vio_method_handle_t *(*csync_method_creat_fn)(const char *durl, mode_t mode);
typedef int (*csync_method_close_fn)(csync_vio_method_handle_t *fhandle);
@ -79,6 +79,7 @@ typedef int (*csync_method_utimes_fn)(const char *uri, const struct timeval time
struct csync_vio_method_s {
size_t method_table_size; /* Used for versioning */
csync_method_get_capabilities_fn get_capabilities;
csync_method_get_file_id_fn get_file_id;
csync_method_open_fn open;
csync_method_creat_fn creat;
csync_method_close_fn close;

6
tests/csync_tests/check_csync_exclude.c
View File

@ -46,6 +46,12 @@ START_TEST (check_csync_excluded)
fail_unless(csync_excluded(csync, ".kde/share/config/kwin.eventsrc") == 0, NULL);
fail_unless(csync_excluded(csync, ".kde4/cache-maximegalon/cache1.txt") == 1, NULL);
fail_unless(csync_excluded(csync, ".mozilla/plugins") == 1, NULL);
/* test for finding patterns in subdirs. .beagle is defined as a pattern and has to
* be found in top dir as well as in directories underneath. */
fail_unless(csync_excluded(csync, ".beagle") == 1, NULL);
fail_unless(csync_excluded(csync, "foo/.beagle") == 1, NULL);
fail_unless(csync_excluded(csync, "bar/foo/.beagle") == 1, NULL);
}
END_TEST

8
tests/csync_tests/check_csync_statedb_query.c
View File

@ -36,8 +36,8 @@ static void setup_db(void) {
fail_unless(csync_statedb_create_tables(csync) == 0, "Setup failed");
stmt = sqlite3_mprintf("INSERT INTO metadata"
"(phash, pathlen, path, inode, uid, gid, mode, modtime) VALUES"
"(%lu, %d, '%q', %d, %d, %d, %d, %lu);",
"(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES"
"(%lu, %d, '%q', %d, %d, %d, %d, %lu, %d, %lu);",
42,
42,
"It's a rainy day",
@ -45,7 +45,9 @@ static void setup_db(void) {
42,
42,
42,
42);
42,
2,
43);
fail_if(csync_statedb_insert(csync, stmt) < 0, NULL);
sqlite3_free(stmt);

2
tests/csync_tests/check_csync_treewalk.c
View File

@ -100,7 +100,7 @@ START_TEST (check_csync_treewalk_local_with_filter)
fail_if(file_count != 2, "Local File count filtered (EVAL) not correct: %d", file_count);
file_count = 0;
fail_if(csync_walk_local_tree(csync, &visitor, CSYNC_INSTRUCTION_EVAL | CSYNC_INSTRUCTION_REMOVE) < 0, "Local walk");
fail_if(file_count != 2, "Local File count filtered (EVAL) not correct: %d", file_count);
fail_if(file_count != 2, "Local File count filtered (EVAL|REMOVE) not correct: %d", file_count);
file_count = 0;
fail_if(csync_walk_local_tree(csync, &visitor, CSYNC_INSTRUCTION_RENAME) < 0, "Local walk");
fail_if(file_count != 0, "Local File count filtered (RENAME) not correct");

2
tests/csync_tests/check_csync_update.c
View File

@ -153,7 +153,7 @@ START_TEST (check_csync_detect_update_db_none)
/* the instruction should be set to new */
st = c_rbtree_node_data(csync->local.tree->root);
fail_unless(st->instruction == CSYNC_INSTRUCTION_NONE, "instruction is %s", csync_instruction_str(st->instruction));
fail_unless(st->instruction == CSYNC_INSTRUCTION_EVAL, "instruction is %s", csync_instruction_str(st->instruction));
/* set the instruction to UPDATED that it gets written to the statedb */
st->instruction = CSYNC_INSTRUCTION_UPDATED;

2075
tests/ownCloud/HTTP/DAV.pm Normal file
View File

File diff suppressed because it is too large Load Diff

2
tests/ownCloud/mocka/CMakeLists.txt
View File

@ -13,8 +13,6 @@ include_directories(
add_executable(ocmod_test ocmod_test.c)
add_executable(md5_test md5_test.c)
target_link_libraries(ocmod_test ${CMOCKA_LIBRARIES} ${NEON_LIBRARIES} ${CSYNC_LIBRARY} )
target_link_libraries(md5_test ${CMOCKA_LIBRARIES} ${CSYNC_LIBRARY} )

68
tests/ownCloud/mocka/md5_test.c
View File

@ -1,68 +0,0 @@
/*
* Copyright 2008 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define _GNU_SOURCE /* See feature_test_macros(7) */
#include <stdio.h>
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <iniparser.h>
#include <std/c_path.h>
#include <csync_util.h>
#include <cmocka.h>
#include "config_test.h"
static void test_md5_buffer(void **state) {
const char *t1 = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.";
const char *t2 = "This is a nice md5 test";
const char *t3 = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet."
"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.";
(void) state;
assert_string_equal( csync_buffer_md5(t1, strlen(t1)), "dbcb1ed05ecf975f532604f2d2000246");
assert_string_equal( csync_buffer_md5(t2, strlen(t2)), "65236ae09754bca371fb869384040141");
assert_string_equal( csync_buffer_md5(t3, strlen(t3)), "651f37892a9df60ed087bc7a1c660fec");
}
static void test_md5_files(void **state) {
char path[255];
(void) state;
strcpy(path, TESTFILES_DIR);
strcat(path, "test.txt");
assert_string_equal( csync_file_md5(path), "f3971ce599093756e6018513d0835134");
strcpy(path, TESTFILES_DIR);
strcat(path, "red_is_the_rose.jpg");
assert_string_equal( csync_file_md5(path), "baf8eeb2a36af94f033fa0094c50c2d5");
}
int main(void) {
const UnitTest tests[] = {
unit_test(test_md5_buffer),
unit_test(test_md5_files)
};
return run_tests(tests);
}

27
tests/ownCloud/mocka/ocmod_test.c
View File

@ -1,17 +1,15 @@
/*
* Copyright 2008 Google Inc.
* Copyright 2012 Klaas Freitag <freitag@owncloud.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* * This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
@ -99,7 +97,7 @@ static void connect_test_success(void **state) {
(void) state;
strcpy(buf, TEST_CONFIG_DIR);
strcat(buf, "test.cfg");
strcat(buf, "mockatest.cfg");
assert_true( load_oc_config( buf ));
@ -134,6 +132,7 @@ static void fetch_a_context(void **state) {
assert_true( fetchCtx->currResource->name != NULL );
printf( " %s -> %s\n", fetchCtx->currResource->uri, fetchCtx->currResource->name );
printf( " MD5: %s\n", fetchCtx->currResource->md5);
fetchCtx->currResource = fetchCtx->currResource->next;
}
}
@ -299,11 +298,13 @@ int main(void) {
const UnitTest tests[] = {
unit_test(null_test_success),
unit_test(connect_test_success),
unit_test(fetch_a_context),
unit_test_setup_teardown(test_setup_dirs, setup_toplevel_dir, teardown_toplevel_dir),
unit_test(test_upload_files),
unit_test(fetch_a_context),
unit_test(test_download_files),
};
srand(time(NULL));
return run_tests(tests);
}

6
tests/ownCloud/mockatest.cfg.in Normal file
View File

@ -0,0 +1,6 @@
[global]
user = 'joe'
pwd = 'secret'
host = 'localhost/owncloud'

46
tests/ownCloud/t1.pl
View File

@ -7,6 +7,8 @@
# Copyright (C) by Klaas Freitag <freitag@owncloud.com>
#
use lib ".";
use Carp::Assert;
use Data::Dumper;
use HTTP::DAV;
@ -110,7 +112,7 @@ sub csync( $$ )
my $cmd = "LD_LIBRARY_PATH=$ld_libpath $csync $local $url";
print "Starting: $cmd\n";
system( $cmd );
system( $cmd ) == 0 or die("CSync died!\n");
}
#
@ -192,9 +194,11 @@ sub glob_put( $$$ )
foreach my $lfile( @puts ) {
if( $lfile =~ /.*\/(.+)$/g ) {
my $rfile = $1;
my $puturl = "$target/$rfile";
my $puturl = "$target"."$rfile";
print " *** Putting $lfile to $puturl\n";
$d->put( -local=>$lfile, -url=> $puturl );
if( ! $d->put( -local=>$lfile, -url=> $puturl ) ) {
print " ### FAILED to put: ". $d->message . '\n';
}
}
}
}
@ -205,40 +209,51 @@ my $d = HTTP::DAV->new();
$d->credentials( -url=> $owncloud, -realm=>"ownCloud",
-user=> $user,
-pass=> $passwd );
$d->DebugLevel(3);
# $d->DebugLevel(3);
my $remoteDir = "t1/";
my $remoteDir = sprintf( "t1-%#.3o/", rand(1000) );
print "Working in remote dir $remoteDir\n";
remoteDir( $d, $remoteDir );
# $d->open("localhost/oc/files/webdav.php/t1");
print "Copy some files to the remote location\n";
remoteDir( $d, $remoteDir . "remoteToLocal1" );
remoteDir( $d, $remoteDir . "remoteToLocal1/rtl1");
remoteDir( $d, $remoteDir . "remoteToLocal1/rtl1/rtl11");
remoteDir( $d, $remoteDir . "remoteToLocal1/rtl2");
# put some files remote.
glob_put( $d, 'toremote1/*', $owncloud . $remoteDir . "remoteToLocal1/" );
glob_put( $d, 'toremote1/rtl1/*', $owncloud . $remoteDir . "remoteToLocal1/rtl1/" );
glob_put( $d, 'toremote1/rtl1/rtl11/*', $owncloud . $remoteDir . "remoteToLocal1/rtl1/rtl11/" );
glob_put( $d, 'toremote1/rtl2/*', $owncloud . $remoteDir . "remoteToLocal1/rtl2/" );
my $localDir = "./t1";
print "Create the local sync dir $localDir\n";
createLocalDir( $localDir );
# call csync, sync local t1 to remote t1
csync( $localDir, $remoteDir );
print "\nNow assertions:\n";
# Check if the files from toremote1 are now in t1/remoteToLocal1
# they should have taken the way via the ownCloud.
print "Assert the local file copy\n";
assertLocalDirs( "./toremote1", "$localDir/remoteToLocal1" );
# Check if the synced files from ownCloud have the same timestamp as the local ones.
print "\nNow assert remote 'toremote1' with local \"$localDir/remoteToLocal1\" :\n";
assertLocalAndRemoteDir( $d, "$localDir/remoteToLocal1", $remoteDir . "remoteToLocal1" );
# remove a local file.
print "\nRemove a local file\n";
unlink( "$localDir/remoteToLocal1/kernelcrash.txt" );
csync( $localDir, $remoteDir );
assertLocalAndRemoteDir( $d, "$localDir/remoteToLocal1", $remoteDir . "remoteToLocal1" );
# add local files to a new dir1
print "\nAdd some more files to local:\n";
my $locDir = $localDir . "/fromLocal1";
mkdir( $locDir );
@ -248,15 +263,30 @@ foreach my $file ( <./tolocal1/*> ) {
copy( $file, $locDir );
}
csync( $localDir, $remoteDir );
print "\nAssert local and remote dirs.\n";
assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" );
# move a local file
print "\nMove a file locally.\n";
move( "$locDir/kramer.jpg", "$locDir/oldtimer.jpg" );
csync( $localDir, $remoteDir );
assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" );
# move a local directory.
print "\nMove a local directory.\n";
move( "$localDir/remoteToLocal1/rtl1", "$localDir/remoteToLocal1/rtlX");
csync( $localDir, $remoteDir );
assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" );
# remove a local dir
print "\nRemove a local directory.\n";
localCleanup( "$localDir/remoteToLocal1/rtlX" );
csync( $localDir, $remoteDir );
assertLocalAndRemoteDir( $d, $locDir, $remoteDir . "fromLocal1" );
print "\n###########################################\n";
print " all cool - tests succeeded.\n";
print " all cool - tests succeeded in $remoteDir.\n";
print "###########################################\n";
print "\nInterrupt before cleanup in 4 seconds...\n";

BIN
tests/ownCloud/testfiles/red_is_the_rose.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 149 KiB

11
tests/ownCloud/testfiles/test.txt Normal file
View File

@ -0,0 +1,11 @@
Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.
Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat.
Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis.
At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur

1
tests/ownCloud/toremote1/rtl1/La cédille Normal file
View File

@ -0,0 +1 @@
A new text.

BIN
tests/ownCloud/toremote1/rtl11/quitte.pdf Normal file
View File

Binary file not shown.

BIN
tests/ownCloud/toremote1/rtl11/red_is_the_rose.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 149 KiB

BIN
tests/ownCloud/toremote1/rtl2/kb1.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 325 KiB

1
tests/ownCloud/toremote1/rtl2/émettre.xls Normal file
View File

@ -0,0 +1 @@
some content.