Merge remote-tracking branch 'ogoffart/davbf' into dav

This commit is contained in:
Klaas Freitag 2013-03-14 13:36:55 +01:00
commit c263532ba3
9 changed files with 332 additions and 42 deletions

View File

@ -164,6 +164,8 @@ struct dav_session_s {
CSYNC *csync_ctx;
void *userdata;
csync_hbf_info_t *chunk_info;
};
/* The list of properties that is fetched in PropFind on a collection */
@ -1666,6 +1668,11 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
rc = 1;
} else {
Hbf_State state = hbf_splitlist(trans, fd);
if (dav_session.chunk_info && dav_session.chunk_info->transfer_id) {
DEBUG_WEBDAV("We have chunk info %d %d ", dav_session.chunk_info->start_id, dav_session.chunk_info->transfer_id);
trans->start_id = dav_session.chunk_info->start_id;
trans->transfer_id = dav_session.chunk_info->transfer_id;
}
if( state == HBF_SUCCESS ) {
chunked_total_size = sb.st_size;
/* Transfer all the chunks through the HTTP session using PUT. */
@ -1675,8 +1682,13 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
if ( state != HBF_SUCCESS ) {
error_string = hbf_error_string(state);
rc = 1;
if (dav_session.chunk_info) {
dav_session.chunk_info->start_id = trans->start_id;
dav_session.chunk_info->transfer_id = trans->transfer_id;
}
}
}
hbf_free_transfer(trans);
}
if (_progresscb) {
ne_set_notifier(dav_session.ctx, 0, 0);
@ -1730,7 +1742,9 @@ static int owncloud_sendfile(csync_vio_method_handle_t *src, csync_vio_method_ha
set_errno_from_neon_errcode(neon_stat);
DEBUG_WEBDAV("Error GET: Neon: %d, errno %d", neon_stat, errno);
rc = -1;
error_string = dav_session.error_string;
error_code = errno;
rc = 1;
} else {
status = ne_get_status( write_ctx->req );
DEBUG_WEBDAV("GET http result %d (%s)", status->code, status->reason_phrase ? status->reason_phrase : "<empty");
@ -2115,6 +2129,10 @@ static int owncloud_set_property(const char *key, void *data) {
dav_session.csync_ctx = data;
return 0;
}
if( c_streq(key, "hbf_info")) {
dav_session.chunk_info = (csync_hbf_info_t *)(data);
return 0;
}
return -1;
}

View File

@ -738,6 +738,12 @@ int csync_destroy(CSYNC *ctx) {
}
#endif
while (ctx->progress) {
csync_progressinfo_t *next = ctx->progress->next;
csync_statedb_free_progressinfo(ctx->progress);
ctx->progress = next;
}
/* destroy the rbtrees */
if (c_rbtree_size(ctx->local.tree) > 0) {
c_rbtree_destroy(ctx->local.tree, _tree_destructor);

View File

@ -38,4 +38,10 @@ int csync_fnmatch(__const char *__pattern, __const char *__name, int __flags);
*/
CSYNC_ERROR_CODE csync_errno_to_csync_error(CSYNC_ERROR_CODE default_err);
struct csync_hbf_info_s {
int start_id;
int transfer_id;
};
typedef struct csync_hbf_info_s csync_hbf_info_t;
#endif /* _CSYNC_MISC_H */

View File

@ -143,6 +143,8 @@ struct csync_s {
uid_t euid;
} pwd;
struct csync_progressinfo_s *progress;
/* replica we are currently walking */
enum csync_replica_e current;

View File

@ -32,12 +32,14 @@
#include "csync_private.h"
#include "csync_propagate.h"
#include "csync_statedb.h"
#include "vio/csync_vio.h"
#include "c_jhash.h"
#define CSYNC_LOG_CATEGORY_NAME "csync.propagator"
#include "csync_log.h"
#include "csync_util.h"
#include "csync_misc.h"
static int _csync_cleanup_cmp(const void *a, const void *b) {
csync_file_stat_t *st_a, *st_b;
@ -69,6 +71,28 @@ static void _store_id_update(CSYNC *ctx, csync_file_stat_t *st) {
}
}
/* Record the error in the ctx->progress
pi may be a previous csync_progressinfo_t from the database.
If pi is NULL, a new one is created, else it is re-used
*/
static void _csync_record_error(CSYNC *ctx, csync_file_stat_t *st, csync_progressinfo_t *pi) {
if (pi) {
pi->error++;
} else {
pi = c_malloc(sizeof(csync_progressinfo_t));
pi->chunk = 0;
pi->transferId = 0;
pi->tmpfile = NULL;
pi->md5 = st->md5 ? c_strdup(st->md5) : NULL;
pi->modtime = st->modtime;
pi->phash = st->phash;
pi->error = 1;
}
pi->next = ctx->progress;
ctx->progress = pi;
}
static bool _push_to_tmp_first(CSYNC *ctx)
{
if( !ctx ) return true;
@ -131,6 +155,21 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
int count = 0;
int flags = 0;
csync_hbf_info_t hbf_info = { 0, 0 };
csync_progressinfo_t *pi = NULL;
pi = csync_statedb_get_progressinfo(ctx, st->phash, st->modtime, st->md5);
if (pi && pi->error > 3) {
rc = 1;
goto out;
}
if (pi) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"continuation: %d %d",
pi->chunk, pi->transferId );
hbf_info.start_id = pi->chunk;
hbf_info.transfer_id = pi->transferId;
}
rep_bak = ctx->replica;
switch (ctx->current) {
@ -188,6 +227,17 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
}
if (_push_to_tmp_first(ctx)) {
int re = 0;
if (pi && pi->tmpfile && pi->tmpfile[0] && _push_to_tmp_first(ctx)) {
turi = c_strdup(pi->tmpfile);
/* Try to see if we can resume. */
ctx->replica = drep;
dfp = csync_vio_open(ctx, turi, O_APPEND|O_NOCTTY, 0);
if (dfp) {
goto start_fd_based;
}
}
/* create the temporary file name */
#ifdef _WIN32
if (asprintf(&turi, "%s.~XXXXXX", duri) < 0) {
@ -300,19 +350,66 @@ static int _csync_push_file(CSYNC *ctx, csync_file_stat_t *st) {
/* copy file */
if( _use_fd_based_push(ctx) ) {
if (ctx->current == REMOTE_REPLICA)
csync_win32_set_file_hidden(turi, true);
start_fd_based:
if (ctx->current == REMOTE_REPLICA) {
csync_win32_set_file_hidden(turi, true);
}
if (!_push_to_tmp_first(ctx)) {
csync_vio_set_property(ctx, "hbf_info", &hbf_info);
}
rc = csync_vio_sendfile( ctx, sfp, dfp );
if (ctx->current == REMOTE_REPLICA)
if (ctx->current == REMOTE_REPLICA) {
csync_win32_set_file_hidden(turi, false);
}
if( rc != 0 ) {
strerror_r(errno, errbuf, sizeof(errbuf));
CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR,
"file: %s, command: sendfile, error: %s from errno %d",
suri, errbuf, errno);
if (_push_to_tmp_first(ctx)) {
csync_vio_file_stat_t* sb = csync_vio_file_stat_new();
if (csync_vio_stat(ctx, turi, sb) == 0 && sb->size > 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"keeping tmp file: %s", turi);
if (!pi) {
pi = c_malloc(sizeof(csync_progressinfo_t));
pi->error = 0;
pi->transferId = 0;
pi->md5 = st->md5 ? c_strdup(st->md5) : NULL;
pi->modtime = st->modtime;
pi->phash = st->phash;
} else {
SAFE_FREE(pi->tmpfile);
}
pi->chunk = 0;
pi->tmpfile = turi;
pi->error <<= 1;
turi = NULL;
}
csync_vio_file_stat_destroy(sb);
} else {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,
"remember chunk: %d (transfer id %d )", hbf_info.start_id, hbf_info.transfer_id);
if (!pi) {
pi = c_malloc(sizeof(csync_progressinfo_t));
pi->error = 0;
pi->md5 = st->md5 ? c_strdup(st->md5) : NULL;
pi->modtime = st->modtime;
pi->phash = st->phash;
pi->tmpfile = NULL;
} else {
SAFE_FREE(pi->tmpfile);
}
pi->transferId = hbf_info.transfer_id;
pi->chunk = hbf_info.start_id;
csync_vio_set_property(ctx, "hbf_info", 0);
}
goto out;
}
} else {
@ -527,14 +624,20 @@ out:
/* set instruction for the statedb merger */
if (rc != 0) {
st->instruction = CSYNC_INSTRUCTION_ERROR;
if (turi != NULL) {
if (_push_to_tmp_first(ctx)) {
/* Remove the tmp file in error case. */
csync_vio_unlink(ctx, turi);
if (_push_to_tmp_first(ctx)) {
if (turi != NULL) {
/* Remove the tmp file in error case. */
csync_vio_unlink(ctx, turi);
}
}
if (rc != 123) {
_csync_record_error(ctx, st, pi);
pi = NULL;
}
}
csync_statedb_free_progressinfo(pi);
SAFE_FREE(prev_tdir);
SAFE_FREE(suri);
SAFE_FREE(duri);
@ -687,6 +790,13 @@ static int _csync_rename_file(CSYNC *ctx, csync_file_stat_t *st) {
const char *tmd5 = NULL;
c_rbnode_t *node = NULL;
csync_progressinfo_t *pi = NULL;
pi = csync_statedb_get_progressinfo(ctx, st->phash, st->modtime, st->md5);
if (pi && pi->error > 3) {
rc = 1;
goto out;
}
switch (ctx->current) {
case REMOTE_REPLICA:
if( !(st->path && st->destpath) ) {
@ -774,8 +884,12 @@ out:
/* set instruction for the statedb merger */
if (rc != 0) {
st->instruction = CSYNC_INSTRUCTION_NONE;
_csync_record_error(ctx, st, pi);
pi = NULL;
}
csync_statedb_free_progressinfo(pi);
return rc;
}
@ -805,6 +919,14 @@ static int _csync_remove_file(CSYNC *ctx, csync_file_stat_t *st) {
char *uri = NULL;
int rc = -1;
csync_progressinfo_t *pi = NULL;
pi = csync_statedb_get_progressinfo(ctx, st->phash, st->modtime, st->md5);
if (pi && pi->error > 3) {
rc = 1;
goto out;
}
switch (ctx->current) {
case LOCAL_REPLICA:
if (asprintf(&uri, "%s/%s", ctx->local.uri, st->path) < 0) {
@ -855,8 +977,11 @@ out:
if (rc != 0) {
/* Write file to statedb, to try to sync again on the next run. */
st->instruction = CSYNC_INSTRUCTION_NONE;
_csync_record_error(ctx, st, pi);
pi = NULL;
}
csync_statedb_free_progressinfo(pi);
return rc;
}
@ -868,6 +993,13 @@ static int _csync_new_dir(CSYNC *ctx, csync_file_stat_t *st) {
struct timeval times[2];
int rc = -1;
csync_progressinfo_t *pi = NULL;
pi = csync_statedb_get_progressinfo(ctx, st->phash, st->modtime, st->md5);
if (pi && pi->error > 3) {
rc = 1;
goto out;
}
replica_bak = ctx->replica;
switch (ctx->current) {
@ -952,8 +1084,11 @@ out:
/* set instruction for the statedb merger */
if (rc != 0) {
st->instruction = CSYNC_INSTRUCTION_ERROR;
_csync_record_error(ctx, st, pi);
pi = NULL;
}
csync_statedb_free_progressinfo(pi);
return rc;
}
@ -965,6 +1100,13 @@ static int _csync_sync_dir(CSYNC *ctx, csync_file_stat_t *st) {
struct timeval times[2];
int rc = -1;
csync_progressinfo_t *pi = NULL;
pi = csync_statedb_get_progressinfo(ctx, st->phash, st->modtime, st->md5);
if (pi && pi->error > 3) {
rc = 1;
goto out;
}
replica_bak = ctx->replica;
switch (ctx->current) {
@ -1032,8 +1174,11 @@ out:
/* set instruction for the statedb merger */
if (rc != 0) {
st->instruction = CSYNC_INSTRUCTION_ERROR;
_csync_record_error(ctx, st, pi);
pi = NULL;
}
csync_statedb_free_progressinfo(pi);
return rc;
}
@ -1347,6 +1492,7 @@ static int _csync_propagation_cleanup(CSYNC *ctx) {
static int _csync_propagation_file_visitor(void *obj, void *data) {
csync_file_stat_t *st = NULL;
CSYNC *ctx = NULL;
int rc = 0;
st = (csync_file_stat_t *) obj;
ctx = (CSYNC *) data;
@ -1357,32 +1503,32 @@ static int _csync_propagation_file_visitor(void *obj, void *data) {
case CSYNC_FTW_TYPE_FILE:
switch (st->instruction) {
case CSYNC_INSTRUCTION_NEW:
if (_csync_new_file(ctx, st) < 0) {
if ((rc = _csync_new_file(ctx, st)) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL NEW: %s",st->path);
goto err;
}
break;
case CSYNC_INSTRUCTION_RENAME:
if (_csync_rename_file(ctx, st) < 0) {
if ((rc = _csync_rename_file(ctx, st)) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL RENAME: %s",st->path);
goto err;
}
break;
case CSYNC_INSTRUCTION_SYNC:
if (_csync_sync_file(ctx, st) < 0) {
if ((rc = _csync_sync_file(ctx, st)) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL SYNC: %s",st->path);
goto err;
}
break;
case CSYNC_INSTRUCTION_REMOVE:
if (_csync_remove_file(ctx, st) < 0) {
if ((rc = _csync_remove_file(ctx, st)) < 0) {
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"FAIL REMOVE: %s",st->path);
goto err;
}
break;
case CSYNC_INSTRUCTION_CONFLICT:
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE,"case CSYNC_INSTRUCTION_CONFLICT: %s",st->path);
if (_csync_conflict_file(ctx, st) < 0) {
if ((rc = _csync_conflict_file(ctx, st)) < 0) {
goto err;
}
break;
@ -1401,7 +1547,7 @@ static int _csync_propagation_file_visitor(void *obj, void *data) {
break;
}
return 0;
return rc;
err:
return -1;
}

View File

@ -251,6 +251,11 @@ int csync_statedb_write(CSYNC *ctx) {
return -1;
}
/* progress info */
if (csync_statedb_write_progressinfo(ctx, ctx->progress) < 0) {
return -1;
}
return 0;
}
@ -360,6 +365,24 @@ int csync_statedb_create_tables(CSYNC *ctx) {
return -1;
}
c_strlist_destroy(result);
result = csync_statedb_query(ctx,
"CREATE TABLE IF NOT EXISTS progress("
"phash INTEGER(8),"
"modtime INTEGER(8),"
"md5 VARCHAR(32),"
"chunk INTEGER(4),"
"transferid INTEGER(4),"
"error_count INTEGER(8),"
"tmpfile VARCHAR(4096),"
"PRIMARY KEY(phash)"
");"
);
if (result == NULL) {
return -1;
}
c_strlist_destroy(result);
/* write the version table. */
stmt = sqlite3_mprintf( "INSERT INTO version (major, minor, patch) VALUES (%d, %d, %d);",
@ -372,6 +395,7 @@ int csync_statedb_create_tables(CSYNC *ctx) {
return -1;
}
sqlite3_free(stmt);
return 0;
}
@ -386,6 +410,14 @@ int csync_statedb_drop_tables(CSYNC *ctx) {
return -1;
}
c_strlist_destroy(result);
result = csync_statedb_query(ctx,
"DROP TABLE IF EXISTS progress;"
);
if (result == NULL) {
return -1;
}
c_strlist_destroy(result);
return 0;
}
@ -844,4 +876,77 @@ int csync_statedb_insert(CSYNC *ctx, const char *statement) {
return sqlite3_last_insert_rowid(ctx->statedb.db);
}
csync_progressinfo_t* csync_statedb_get_progressinfo(CSYNC *ctx, uint64_t phash, uint64_t modtime, const char* md5) {
char *stmt = NULL;
csync_progressinfo_t *ret = NULL;
c_strlist_t *result = NULL;
if( ! csync_get_statedb_exists(ctx)) return ret;
stmt = sqlite3_mprintf("SELECT error_count, chunk, transferid, tmpfile FROM progress WHERE phash='%llu' AND modtime='%lld' AND md5='%q'",
(long long unsigned int) phash, (long long signed int) modtime, md5);
if (!stmt) return ret;
result = csync_statedb_query(ctx, stmt);
sqlite3_free(stmt);
if (result == NULL) {
return NULL;
}
if (result->count == 4) {
ret = c_malloc(sizeof(csync_progressinfo_t));
if (!ret) goto out;
ret->next = NULL;
ret->error = atoi(result->vector[0]);
ret->chunk = atoi(result->vector[1]);
ret->transferId = atoi(result->vector[2]);
ret->tmpfile = c_strdup(result->vector[3]);
ret->md5 = md5 ? c_strdup(md5) : NULL;
ret->modtime = modtime;
ret->phash = phash;
}
out:
c_strlist_destroy(result);
return ret;
}
void csync_statedb_free_progressinfo(csync_progressinfo_t* pi)
{
if (!pi) return;
SAFE_FREE(pi->md5);
SAFE_FREE(pi->tmpfile);
SAFE_FREE(pi);
}
int csync_statedb_write_progressinfo(CSYNC* ctx, csync_progressinfo_t* pi)
{
int rc = 0;
char *stmt = NULL;
while (rc > -1 && pi) {
stmt = sqlite3_mprintf("INSERT INTO progress "
"(phash, modtime, md5, chunk, transferid, error_count, tmpfile) VALUES"
"(%llu, %lld, '%q', %d, %d, %d, '%q');",
(long long signed int) pi->phash,
(long long int) pi->modtime,
pi->md5,
pi->chunk,
pi->transferId,
pi->error,
pi->tmpfile);
if (stmt == NULL) {
return -1;
}
CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s" , stmt);
rc = csync_statedb_insert(ctx, stmt);
sqlite3_free(stmt);
pi = pi->next;
}
return 0;
}
/* vim: set ts=8 sw=2 et cindent: */

View File

@ -110,6 +110,22 @@ int csync_statedb_drop_tables(CSYNC *ctx);
int csync_statedb_insert_metadata(CSYNC *ctx);
typedef struct csync_progressinfo_s {
struct csync_progressinfo_s *next;
uint64_t phash;
uint64_t modtime;
char *md5;
int error;
int chunk;
int transferId;
char *tmpfile;
} csync_progressinfo_t;
csync_progressinfo_t *csync_statedb_get_progressinfo(CSYNC *ctx, uint64_t phash, uint64_t modtime, const char *md5);
void csync_statedb_free_progressinfo(csync_progressinfo_t *pi);
int csync_statedb_write_progressinfo(CSYNC *ctx, csync_progressinfo_t *pi);
/**
* }@
*/

View File

@ -80,6 +80,7 @@ hbf_transfer_t *hbf_init_transfer( const char *dest_uri ) {
transfer->url = strdup(dest_uri);
transfer->status_code = 200;
transfer->error_string = NULL;
transfer->start_id = 0;
return transfer;
}
@ -131,6 +132,7 @@ Hbf_State hbf_splitlist(hbf_transfer_t *transfer, int fd ) {
transfer->block_arr = calloc(num_blocks, sizeof(hbf_block_t*));
transfer->block_cnt = num_blocks;
transfer->transfer_id = transfer_id(&sb);
transfer->start_id = 0;
for( cnt=0; cnt < num_blocks; cnt++ ) {
/* allocate a block struct and fill */
@ -206,9 +208,10 @@ char* get_transfer_url( hbf_transfer_t *transfer, int indx ) {
}
static int dav_request( ne_request *req, int fd, hbf_block_t *blk ) {
Hbf_State state = HBF_SUCCESS;
Hbf_State state = HBF_TRANSFER_SUCCESS;
int res;
const ne_status *req_status = NULL;
const char *etag = NULL;
if( ! (blk && req) ) return HBF_PARAM_FAIL;
@ -222,6 +225,9 @@ static int dav_request( ne_request *req, int fd, hbf_block_t *blk ) {
case NE_OK:
state = HBF_SUCCESS;
blk->state = HBF_TRANSFER_SUCCESS;
etag = ne_get_response_header(req, "ETag");
if (etag && etag[0])
state = HBF_SUCCESS;
break;
case NE_AUTH:
state = HBF_AUTH_FAIL;
@ -254,7 +260,7 @@ static int dav_request( ne_request *req, int fd, hbf_block_t *blk ) {
}
Hbf_State hbf_transfer( ne_session *session, hbf_transfer_t *transfer, const char *verb ) {
Hbf_State state = HBF_SUCCESS;
Hbf_State state = HBF_TRANSFER_SUCCESS;
int cnt;
int fail_cnt = 0;
@ -269,20 +275,20 @@ Hbf_State hbf_transfer( ne_session *session, hbf_transfer_t *transfer, const cha
}
for( cnt=0; state == HBF_SUCCESS && cnt < transfer->block_cnt; cnt++ ) {
hbf_block_t *block = transfer->block_arr[cnt];
int block_id = (cnt + transfer->start_id) % transfer->block_cnt;
Hbf_State block_state = HBF_SUCCESS;
hbf_block_t *block = transfer->block_arr[block_id];
char *transfer_url = NULL;
if( ! block ) state = HBF_PARAM_FAIL;
if( state == HBF_SUCCESS ) {
transfer_url = get_transfer_url( transfer, cnt );
if( state == HBF_TRANSFER_SUCCESS ) {
transfer_url = get_transfer_url( transfer, block_id );
if( ! transfer_url ) {
state = HBF_PARAM_FAIL;
}
}
if( state == HBF_SUCCESS ) {
if( state == HBF_TRANSFER_SUCCESS ) {
ne_request *req = ne_request_create(session, "PUT", transfer_url);
if( req ) {
@ -292,20 +298,11 @@ Hbf_State hbf_transfer( ne_session *session, hbf_transfer_t *transfer, const cha
if( block_state != HBF_SUCCESS ) {
if( transfer->error_string ) free( transfer->error_string );
transfer->error_string = strdup( ne_get_error(session) );
/* Set the code of the last transmission. */
transfer->status_code = transfer->block_arr[cnt]->http_result_code;
state = HBF_FAIL;
fail_cnt++;
if( block_state == HBF_FAIL ) {
/* not every problem here is a critical one. Try other parts */
if( transfer->status_code == 405 ) {
/* continue to upload... */
state = HBF_SUCCESS;
}
}
transfer->error_string = strdup( ne_get_error(session) );
transfer->start_id = block_id % transfer->block_cnt;
/* Set the code of the last transmission. */
state = HBF_FAIL;
transfer->status_code = transfer->block_arr[block_id]->http_result_code;
}
ne_request_destroy(req);
} else {
@ -314,13 +311,6 @@ Hbf_State hbf_transfer( ne_session *session, hbf_transfer_t *transfer, const cha
free( transfer_url );
}
}
if( state == HBF_SUCCESS && fail_cnt > 0 ) {
state = HBF_FAIL; /* All failed! */
if( fail_cnt < transfer->block_cnt ) {
/* Not all failed actually. */
state = HBF_PARTIAL_TRANSFER_SUCCESS;
}
}
return state;
}

View File

@ -72,6 +72,7 @@ struct hbf_transfer_s {
int fd;
int transfer_id;
char *url;
int start_id;
int status_code;
char *error_string;