mirrors/urbackup_backend
2
0
Fork 0
mirror of https://github.com/uroni/urbackup_backend.git synced 2025-10-26 11:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Background checkpointing for all databases

Browse Source
This commit is contained in:
Martin 2016-01-18 15:43:00 +01:00
parent 5d9f9db1d1
commit c77a0b49ec
3 changed files with 81 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
urbackupserver/WalCheckpointThread.cpp
View File

@ -22,10 +22,12 @@
#include "../Interface/Server.h"
#include "database.h"
#include "../Interface/Database.h"
#include "../stringtools.h"
#include <memory>
WalCheckpointThread::WalCheckpointThread()
: last_checkpoint_wal_size(0)
WalCheckpointThread::WalCheckpointThread(int64 passive_checkpoint_size, int64 full_checkpoint_size, const std::string& db_fn, DATABASE_ID db_id)
: last_checkpoint_wal_size(0), passive_checkpoint_size(passive_checkpoint_size),
full_checkpoint_size(full_checkpoint_size), db_fn(db_fn), db_id(db_id)
{
}
@ -35,52 +37,29 @@ void WalCheckpointThread::checkpoint()
#ifdef _WIN32
mode=MODE_READ_DEVICE;
#endif
std::auto_ptr<IFile> wal_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db-wal", mode));
std::auto_ptr<IFile> wal_file(Server->openFile(db_fn+"-wal", mode));
if(wal_file.get()!=NULL)
{
int64 wal_size = wal_file->Size();
if (wal_size > 1 * 1024 * 1024 * 1024) //>1GiB
if (wal_size > full_checkpoint_size)
{
wal_file.reset();
passive_checkpoint();
Server->Log("Syncing database...", LL_DEBUG);
Server->Log("Files WAL file "+ db_fn + "-wal greater than "+PrettyPrintBytes(full_checkpoint_size)+". Doing full WAL checkpoint...", LL_INFO);
int rw_mode = MODE_RW;
#ifdef _WIN32
rw_mode = MODE_RW_DEVICE;
#endif
{
std::auto_ptr<IFile> db_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db", rw_mode));
if (db_file.get() != NULL)
{
db_file->Sync();
}
}
Server->Log("Syncing wal file...", LL_DEBUG);
{
std::auto_ptr<IFile> rw_wal_file(Server->openFile("urbackup" + os_file_sep() + "backup_server_files.db-wal", rw_mode));
if (rw_wal_file.get() != NULL)
{
rw_wal_file->Sync();
}
}
Server->Log("Files WAL file greater than 1GiB. Doing full WAL checkpoint...", LL_INFO);
IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES);
IDatabase* db = Server->getDatabase(Server->getThreadID(), db_id);
db->lockForSingleUse();
db->Write("PRAGMA wal_checkpoint(TRUNCATE)");
db->unlockForSingleUse();
Server->Log("Full checkpoint of "+ db_fn + "-wal done.", LL_INFO);
last_checkpoint_wal_size = 0;
}
else if (wal_size - last_checkpoint_wal_size > 100 * 1024 * 1024) //100MB
else if (wal_size - last_checkpoint_wal_size > passive_checkpoint_size)
{
last_checkpoint_wal_size = wal_size;
@ -108,13 +87,45 @@ void WalCheckpointThread::operator()()
}
}
void WalCheckpointThread::sync_database()
{
ScopedBackgroundPrio background_prio;
Server->Log("Syncing database " + db_fn + "...", LL_DEBUG);
int rw_mode = MODE_RW;
#ifdef _WIN32
rw_mode = MODE_RW_DEVICE;
#endif
{
std::auto_ptr<IFile> db_file(Server->openFile(db_fn, rw_mode));
if (db_file.get() != NULL)
{
db_file->Sync();
}
}
Server->Log("Syncing wal file " + db_fn + "-wal...", LL_DEBUG);
{
std::auto_ptr<IFile> rw_wal_file(Server->openFile(db_fn + "-wal", rw_mode));
if (rw_wal_file.get() != NULL)
{
rw_wal_file->Sync();
}
}
}
void WalCheckpointThread::passive_checkpoint()
{
Server->Log("Starting passive WAL checkpoint...", LL_DEBUG);
IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES);
ScopedBackgroundPrio background_prio;
Server->Log("Starting passive WAL checkpoint of "+db_fn+"...", LL_DEBUG);
IDatabase* db = Server->getDatabase(Server->getThreadID(), db_id);
db_results res = db->Read("PRAGMA wal_checkpoint(PASSIVE)");
if (!res.empty())
{
Server->Log("Passive WAL checkpoint completed busy=" + res[0]["busy"] + " checkpointed=" + res[0]["checkpointed"] + " log=" + res[0]["log"], LL_DEBUG);
}
Server->Log("Passive WAL checkpoint of " + db_fn + " completed busy=" + res[0]["busy"] + " checkpointed=" + res[0]["checkpointed"] + " log=" + res[0]["log"], LL_DEBUG);
}
}

9
urbackupserver/WalCheckpointThread.h
View File

@ -5,7 +5,7 @@
class WalCheckpointThread : public IThread
{
public:
WalCheckpointThread();
WalCheckpointThread(int64 passive_checkpoint_size, int64 full_checkpoint_size, const std::string& db_fn, DATABASE_ID db_id);
void checkpoint();
@ -13,7 +13,14 @@ public:
private:
void sync_database();
void passive_checkpoint();
int64 last_checkpoint_wal_size;
int64 passive_checkpoint_size;
int64 full_checkpoint_size;
std::string db_fn;
DATABASE_ID db_id;
};

34
urbackupserver/dllmain.cpp
View File

@ -166,8 +166,11 @@ void open_server_database(bool init_db)
{
writestring(get_backup_server_db_data(), "urbackup/backup_server.db");
}
str_map params;
params["wal_autocheckpoint"] = "0";
if(! Server->openDatabase("urbackup/backup_server.db", URBACKUPDB_SERVER) )
if(! Server->openDatabase("urbackup/backup_server.db", URBACKUPDB_SERVER, params) )
{
Server->Log("Couldn't open Database backup_server.db. Exiting. Expecting database at \"" +
Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server.db\"", LL_ERROR);
@ -183,8 +186,6 @@ void open_server_database(bool init_db)
exit(1);
}
str_map params;
params["wal_autocheckpoint"] = "0";
if (!Server->openDatabase("urbackup/backup_server_files.db", URBACKUPDB_SERVER_FILES, params))
{
@ -193,6 +194,8 @@ void open_server_database(bool init_db)
exit(1);
}
Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_FILES, sqlite_data_allocation_chunk_size);
if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES))
{
Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" +
@ -200,13 +203,15 @@ void open_server_database(bool init_db)
exit(1);
}
if (!Server->openDatabase("urbackup/backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL))
if (!Server->openDatabase("urbackup/backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL, params))
{
Server->Log("Couldn't open Database backup_server_link_journal.db. Exiting. Expecting database at \"" +
Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server_link_journal.db\"", LL_ERROR);
exit(1);
}
Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_LINK_JOURNAL, sqlite_data_allocation_chunk_size);
if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_LINK_JOURNAL))
{
Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" +
@ -214,13 +219,15 @@ void open_server_database(bool init_db)
exit(1);
}
if (!Server->openDatabase("urbackup/backup_server_links.db", URBACKUPDB_SERVER_LINKS))
if (!Server->openDatabase("urbackup/backup_server_links.db", URBACKUPDB_SERVER_LINKS, params))
{
Server->Log("Couldn't open Database backup_server_links.db. Exiting. Expecting database at \"" +
Server->getServerWorkingDir() + os_file_sep() + "urbackup" + os_file_sep() + "backup_server_links.db\"", LL_ERROR);
exit(1);
}
Server->setDatabaseAllocationChunkSize(URBACKUPDB_SERVER_LINKS, sqlite_data_allocation_chunk_size);
if (!Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_LINKS))
{
Server->Log("Couldn't open backup server database. Exiting. Expecting database at \"" +
@ -714,10 +721,21 @@ DLLEXPORT void LoadActions(IServer* pServer)
Server->setLogCircularBufferSize(20);
WalCheckpointThread* wal_checkpoint_thread = new WalCheckpointThread();
wal_checkpoint_thread->checkpoint();
WalCheckpointThread* wal_checkpoint_thread = new WalCheckpointThread(100*1024*1024, 1000*1024*1024,
"urbackup"+os_file_sep()+"backup_server_files.db", URBACKUPDB_SERVER_FILES);
Server->createThread(wal_checkpoint_thread, "files checkpoint");
Server->createThread(wal_checkpoint_thread, "WAL checkpoint");
wal_checkpoint_thread = new WalCheckpointThread(10 * 1024 * 1024, 100 * 1024 * 1024,
"urbackup" + os_file_sep() + "backup_server.db", URBACKUPDB_SERVER);
Server->createThread(wal_checkpoint_thread, "main checkpoint");
wal_checkpoint_thread = new WalCheckpointThread(10 * 1024 * 1024, 100 * 1024 * 1024,
"urbackup" + os_file_sep() + "backup_server_link_journal.db", URBACKUPDB_SERVER_LINK_JOURNAL);
Server->createThread(wal_checkpoint_thread, "lnk jour checkpoint");
wal_checkpoint_thread = new WalCheckpointThread(100 * 1024 * 1024, 1000 * 1024 * 1024,
"urbackup" + os_file_sep() + "backup_server_links.db", URBACKUPDB_SERVER_LINKS);
Server->createThread(wal_checkpoint_thread, "lnk checkpoint");
Server->Log("UrBackup Server start up complete.", LL_INFO);
}