From c77a0b49ecb3c2cc98d8be112e0fef4cfcc396ec Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 18 Jan 2016 15:43:00 +0100 Subject: [PATCH] Background checkpointing for all databases --- urbackupserver/WalCheckpointThread.cpp | 83 +++++++++++++++----------- urbackupserver/WalCheckpointThread.h | 9 ++- urbackupserver/dllmain.cpp | 34 ++++++++--- 3 files changed, 81 insertions(+), 45 deletions(-) diff --git a/urbackupserver/WalCheckpointThread.cpp b/urbackupserver/WalCheckpointThread.cpp index 80fdf6fe..3d0f7a34 100644 --- a/urbackupserver/WalCheckpointThread.cpp +++ b/urbackupserver/WalCheckpointThread.cpp @@ -22,10 +22,12 @@ #include "../Interface/Server.h" #include "database.h" #include "../Interface/Database.h" +#include "../stringtools.h" #include -WalCheckpointThread::WalCheckpointThread() - : last_checkpoint_wal_size(0) +WalCheckpointThread::WalCheckpointThread(int64 passive_checkpoint_size, int64 full_checkpoint_size, const std::string& db_fn, DATABASE_ID db_id) + : last_checkpoint_wal_size(0), passive_checkpoint_size(passive_checkpoint_size), + full_checkpoint_size(full_checkpoint_size), db_fn(db_fn), db_id(db_id) { } @@ -35,52 +37,29 @@ void WalCheckpointThread::checkpoint() #ifdef _WIN32 mode=MODE_READ_DEVICE; #endif - std::auto_ptr wal_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db-wal", mode)); + std::auto_ptr wal_file(Server->openFile(db_fn+"-wal", mode)); if(wal_file.get()!=NULL) { int64 wal_size = wal_file->Size(); - if (wal_size > 1 * 1024 * 1024 * 1024) //>1GiB + if (wal_size > full_checkpoint_size) { wal_file.reset(); passive_checkpoint(); - Server->Log("Syncing database...", LL_DEBUG); + Server->Log("Files WAL file "+ db_fn + "-wal greater than "+PrettyPrintBytes(full_checkpoint_size)+". Doing full WAL checkpoint...", LL_INFO); - int rw_mode = MODE_RW; -#ifdef _WIN32 - rw_mode = MODE_RW_DEVICE; -#endif - { - std::auto_ptr db_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db", rw_mode)); - if (db_file.get() != NULL) - { - db_file->Sync(); - } - } - - Server->Log("Syncing wal file...", LL_DEBUG); - - { - std::auto_ptr rw_wal_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db-wal", rw_mode)); - if (rw_wal_file.get() != NULL) - { - rw_wal_file->Sync(); - } - } - - - Server->Log("Files WAL file greater than 1GiB. Doing full WAL checkpoint...", LL_INFO); - - IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES); + IDatabase* db = Server->getDatabase(Server->getThreadID(), db_id); db->lockForSingleUse(); db->Write("PRAGMA wal_checkpoint(TRUNCATE)"); db->unlockForSingleUse(); + Server->Log("Full checkpoint of "+ db_fn + "-wal done.", LL_INFO); + last_checkpoint_wal_size = 0; } - else if (wal_size - last_checkpoint_wal_size > 100 * 1024 * 1024) //100MB + else if (wal_size - last_checkpoint_wal_size > passive_checkpoint_size) { last_checkpoint_wal_size = wal_size; @@ -108,13 +87,45 @@ void WalCheckpointThread::operator()() } } +void WalCheckpointThread::sync_database() +{ + ScopedBackgroundPrio background_prio; + + Server->Log("Syncing database " + db_fn + "...", LL_DEBUG); + + int rw_mode = MODE_RW; +#ifdef _WIN32 + rw_mode = MODE_RW_DEVICE; +#endif + + { + std::auto_ptr db_file(Server->openFile(db_fn, rw_mode)); + if (db_file.get() != NULL) + { + db_file->Sync(); + } + } + + Server->Log("Syncing wal file " + db_fn + "-wal...", LL_DEBUG); + + { + std::auto_ptr rw_wal_file(Server->openFile(db_fn + "-wal", rw_mode)); + if (rw_wal_file.get() != NULL) + { + rw_wal_file->Sync(); + } + } +} + void WalCheckpointThread::passive_checkpoint() { - Server->Log("Starting passive WAL checkpoint...", LL_DEBUG); - IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES); + ScopedBackgroundPrio background_prio; + + Server->Log("Starting passive WAL checkpoint of "+db_fn+"...", LL_DEBUG); + IDatabase* db = Server->getDatabase(Server->getThreadID(), db_id); db_results res = db->Read("PRAGMA wal_checkpoint(PASSIVE)"); if (!res.empty()) { - Server->Log("Passive WAL checkpoint completed busy=" + res[0]["busy"] + " checkpointed=" + res[0]["checkpointed"] + " log=" + res[0]["log"], LL_DEBUG); - } + Server->Log("Passive WAL checkpoint of " + db_fn + " completed busy=" + res[0]["busy"] + " checkpointed=" + res[0]["checkpointed"] + " log=" + res[0]["log"], LL_DEBUG); + } } diff --git a/urbackupserver/WalCheckpointThread.h b/urbackupserver/WalCheckpointThread.h index 4d9a97d7..d9a4f97b 100644 --- a/urbackupserver/WalCheckpointThread.h +++ b/urbackupserver/WalCheckpointThread.h @@ -5,7 +5,7 @@ class WalCheckpointThread : public IThread { public: - WalCheckpointThread(); + WalCheckpointThread(int64 passive_checkpoint_size, int64 full_checkpoint_size, const std::string& db_fn, DATABASE_ID db_id); void checkpoint(); @@ -13,7 +13,14 @@ public: private: + void sync_database(); + void passive_checkpoint(); int64 last_checkpoint_wal_size; + + int64 passive_checkpoint_size; + int64 full_checkpoint_size; + std::string db_fn; + DATABASE_ID db_id; }; \ No newline at end of file diff --git a/urbackupserver/dllmain.cpp b/urbackupserver/dllmain.cpp index 8281b051..97c54ce1 100644 --- a/urbackupserver/dllmain.cpp +++ b/urbackupserver/dllmain.cpp @@ -166,8 +166,11 @@ void open_server_database(bool init_db) { writestring(get_backup_server_db_data(), "urbackup/backup_server.db"); } + + str_map params; + params["wal_autocheckpoint"] = "0"; - if(! Server->openDatabase("urbackup/backup_server.db", URBACKUPDB_SERVER) ) + if(! Server->openDatabase("urbackup/backup_server.db", URBACKUPDB_SERVER, params) ) { Server->Log("Couldn't open Database backup_server.db. Exiting. Expecting database at \"" + Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server.db\"", LL_ERROR); @@ -183,8 +186,6 @@ void open_server_database(bool init_db) exit(1); } - str_map params; - params["wal_autocheckpoint"] = "0"; if (!Server->openDatabase("urbackup/backup_server_files.db", URBACKUPDB_SERVER_FILES, params)) { @@ -193,6 +194,8 @@ void open_server_database(bool init_db) exit(1); } + Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_FILES, sqlite_data_allocation_chunk_size); + if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES)) { Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" + @@ -200,13 +203,15 @@ void open_server_database(bool init_db) exit(1); } - if (!Server->openDatabase("urbackup/backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL)) + if (!Server->openDatabase("urbackup/backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL, params)) { Server->Log("Couldn't open Database backup_server_link_journal.db. Exiting. Expecting database at \"" + Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server_link_journal.db\"", LL_ERROR); exit(1); } + Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_LINK_JOURNAL, sqlite_data_allocation_chunk_size); + if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_LINK_JOURNAL)) { Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" + @@ -214,13 +219,15 @@ void open_server_database(bool init_db) exit(1); } - if (!Server->openDatabase("urbackup/backup_server_links.db", URBACKUPDB_SERVER_LINKS)) + if (!Server->openDatabase("urbackup/backup_server_links.db", URBACKUPDB_SERVER_LINKS, params)) { Server->Log("Couldn't open Database backup_server_links.db. Exiting. Expecting database at \"" + Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server_links.db\"", LL_ERROR); exit(1); } + Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_LINKS, sqlite_data_allocation_chunk_size); + if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_LINKS)) { Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" + @@ -714,10 +721,21 @@ DLLEXPORT void LoadActions(IServer* pServer) Server->setLogCircularBufferSize(20); - WalCheckpointThread* wal_checkpoint_thread = new WalCheckpointThread(); - wal_checkpoint_thread->checkpoint(); + WalCheckpointThread* wal_checkpoint_thread = new WalCheckpointThread(100*1024*1024, 1000*1024*1024, + "urbackup"+os_file_sep()+"backup_server_files.db", URBACKUPDB_SERVER_FILES); + Server->createThread(wal_checkpoint_thread, "files checkpoint"); - Server->createThread(wal_checkpoint_thread, "WAL checkpoint"); + wal_checkpoint_thread = new WalCheckpointThread(10 * 1024 * 1024, 100 * 1024 * 1024, + "urbackup" + os_file_sep() + "backup_server.db", URBACKUPDB_SERVER); + Server->createThread(wal_checkpoint_thread, "main checkpoint"); + + wal_checkpoint_thread = new WalCheckpointThread(10 * 1024 * 1024, 100 * 1024 * 1024, + "urbackup" + os_file_sep() + "backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL); + Server->createThread(wal_checkpoint_thread, "lnk jour checkpoint"); + + wal_checkpoint_thread = new WalCheckpointThread(100 * 1024 * 1024, 1000 * 1024 * 1024, + "urbackup" + os_file_sep() + "backup_server_links.db", URBACKUPDB_SERVER_LINKS); + Server->createThread(wal_checkpoint_thread, "lnk checkpoint"); Server->Log("UrBackup Server start up complete.", LL_INFO); }