mirrors/urbackup_backend
2
0
Fork 0
mirror of https://github.com/uroni/urbackup_backend.git synced 2025-10-26 11:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Throtting to percent of max speed determined by regular probing for max speed

Browse Source
This commit is contained in:
Martin Raiber 2016-11-12 22:23:34 +01:00
parent dff3ab7ae3
commit bb2dcdbca4
14 changed files with 247 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Interface/PipeThrottler.h
View File

@ -9,14 +9,14 @@ class IPipeThrottlerUpdater : public IObject
public:
virtual int64 getUpdateIntervalMs()=0;
virtual size_t getThrottleLimit()=0;
virtual size_t getThrottleLimit(bool& percent_max)=0;
};
class IPipeThrottler : public IObject
{
public:
virtual bool addBytes(size_t n_bytes, bool wait)=0;
virtual void changeThrottleLimit(size_t bps)=0;
virtual void changeThrottleLimit(size_t bps, bool p_percent_max)=0;
virtual void changeThrottleUpdater(IPipeThrottlerUpdater* new_updater)=0;
};

3
Interface/Server.h
View File

@ -104,7 +104,8 @@ public:
virtual ISettingsReader* createDBMemSettingsReader(THREAD_ID tid, DATABASE_ID pIdentifier, const std::string &pTable, const std::string &pSQL = "") = 0;
virtual ISettingsReader* createDBMemSettingsReader(IDatabase *db, const std::string &pTable, const std::string &pSQL = "") = 0;
virtual ISettingsReader* createMemorySettingsReader(const std::string &pData)=0;
virtual IPipeThrottler* createPipeThrottler(size_t bps, IPipeThrottlerUpdater* updater=NULL)=0;
virtual IPipeThrottler* createPipeThrottler(size_t bps, bool percent_max) = 0;
virtual IPipeThrottler* createPipeThrottler(IPipeThrottlerUpdater* updater) = 0;
virtual bool openDatabase(std::string pFile, DATABASE_ID pIdentifier, const str_map& params = str_map(), std::string pEngine="sqlite")=0;
virtual IDatabase* getDatabase(THREAD_ID tid, DATABASE_ID pIdentifier)=0;

99
PipeThrottler.cpp
View File

@ -24,9 +24,14 @@
#define DLOG(x) //x
PipeThrottler::PipeThrottler(size_t bps,
bool percent_max,
IPipeThrottlerUpdater* updater)
: throttle_bps(bps), curr_bytes(0),
lastresettime(0), updater(updater)
: throttle_bps(bps), percent_max(percent_max), curr_bytes(0),
lastresettime(0), updater(updater),
throttle_state(ThrottleState_Probe),
lastprobetime(0), probe_bps(0),
throttle_percent(bps), last_probe_result(0),
probe_interval(10 * 60 * 1000)
{
mutex=Server->createMutex();
lastupdatetime=Server->getTimeMS();
@ -56,17 +61,92 @@ bool PipeThrottler::addBytes(size_t new_bytes, bool wait)
if(updater.get() && update_time_interval>=0 &&
ctime-lastupdatetime>update_time_interval)
{
throttle_bps = updater->getThrottleLimit();
size_t new_throttle_bps = updater->getThrottleLimit(percent_max);
if (percent_max)
{
throttle_percent = new_throttle_bps;
if (throttle_percent == 0)
{
throttle_bps = 0;
}
}
else
{
throttle_bps = new_throttle_bps;
}
lastupdatetime = ctime;
if(throttle_bps==0) return true;
}
if (percent_max &&
throttle_state == ThrottleState_Throttle
&& ctime - lastprobetime > static_cast<int64>(probe_interval))
{
throttle_state = ThrottleState_Probe;
probe_bps = 0;
Server->Log("PROBE Starting probing for max speed");
}
if(ctime-lastresettime>1000)
{
if (percent_max && throttle_state == ThrottleState_Probe)
{
int64 passed_time = ctime - lastresettime;
float bps = (curr_bytes * 1000.f) / passed_time;
if (probe_bps == 0)
{
probe_bps = bps;
}
else
{
float new_probe_bps = 0.9f*probe_bps + 0.1f*bps;
float pdiff = new_probe_bps / probe_bps;
if (pdiff > 0.95f && pdiff < 1.05f)
{
Server->Log("PROBE Probing finished at current speed " + PrettyPrintSpeed(static_cast<size_t>(bps + 0.5f))
+ " last avg " + PrettyPrintSpeed(static_cast<size_t>(probe_bps+0.5f))
+ " curr avg " + PrettyPrintSpeed(static_cast<size_t>(new_probe_bps+0.5f))
+ " pdiff " + convert(pdiff), LL_DEBUG);
lastprobetime = ctime;
throttle_state = ThrottleState_Throttle;
throttle_bps = static_cast<size_t>((static_cast<float>(throttle_percent) / 100)*new_probe_bps + 0.5f);
if (last_probe_result != 0)
{
pdiff = last_probe_result / new_probe_bps;
Server->Log("PROBE Curr probe result " + PrettyPrintBytes(static_cast<size_t>(new_probe_bps+0.5f))
+ " last probe result " + PrettyPrintBytes(static_cast<size_t>(last_probe_result+0.5f))
+ " pdiff " + convert(pdiff), LL_DEBUG);
if (pdiff > 0.95f && pdiff < 1.05f)
{
probe_interval += 10 * 60 * 1000;
Server->Log("PROBE New probe interval: " + PrettyPrintTime(probe_interval), LL_DEBUG);
}
}
last_probe_result = new_probe_bps;
}
else
{
Server->Log("PROBE Probing at current speed " + PrettyPrintSpeed(static_cast<size_t>(bps + 0.5f))
+ " last avg " + PrettyPrintSpeed(static_cast<size_t>(probe_bps+0.5f))
+ " curr avg " + PrettyPrintSpeed(static_cast<size_t>(new_probe_bps+0.5f))
+ " pdiff " + convert(pdiff), LL_DEBUG);
}
probe_bps = new_probe_bps;
}
}
lastresettime=ctime;
curr_bytes=0;
}
if (percent_max && throttle_state == ThrottleState_Probe)
{
return true;
}
curr_bytes+=new_bytes;
int64 passed_time=ctime-lastresettime;
@ -112,11 +192,20 @@ bool PipeThrottler::addBytes(size_t new_bytes, bool wait)
return true;
}
void PipeThrottler::changeThrottleLimit(size_t bps)
void PipeThrottler::changeThrottleLimit(size_t bps, bool p_percent_max)
{
IScopedLock lock(mutex);
throttle_bps=bps;
percent_max = p_percent_max;
if (percent_max)
{
throttle_percent = bps;
}
else
{
throttle_bps = bps;
}
}
void PipeThrottler::changeThrottleUpdater(IPipeThrottlerUpdater* new_updater)

18
PipeThrottler.h
View File

@ -8,22 +8,36 @@ class IMutex;
class PipeThrottler : public IPipeThrottler
{
public:
PipeThrottler(size_t bps, IPipeThrottlerUpdater* updater);
PipeThrottler(size_t bps, bool percent_max, IPipeThrottlerUpdater* updater);
PipeThrottler();
~PipeThrottler(void);
virtual bool addBytes(size_t new_bytes, bool wait);
virtual void changeThrottleLimit(size_t bps);
virtual void changeThrottleLimit(size_t bps, bool p_percent_max);
virtual void changeThrottleUpdater(IPipeThrottlerUpdater* new_updater);
private:
enum ThrottleState
{
ThrottleState_Probe,
ThrottleState_Throttle
};
size_t throttle_bps;
bool percent_max;
int64 update_time_interval;
size_t curr_bytes;
int64 lastresettime;
int64 lastupdatetime;
std::auto_ptr<IPipeThrottlerUpdater> updater;
ThrottleState throttle_state;
int64 lastprobetime;
float probe_bps;
size_t throttle_percent;
float last_probe_result;
size_t probe_interval;
IMutex *mutex;
};

11
Server.cpp
View File

@ -1745,11 +1745,18 @@ void CServer::startupComplete(void)
}
IPipeThrottler* CServer::createPipeThrottler(size_t bps,
IPipeThrottlerUpdater* updater)
bool percent_max)
{
return new PipeThrottler(bps, updater);
return new PipeThrottler(bps, percent_max, NULL);
}
IPipeThrottler* CServer::createPipeThrottler(
IPipeThrottlerUpdater* updater)
{
bool percent_max = false;
size_t bps = updater->getThrottleLimit(percent_max);
return new PipeThrottler(bps, percent_max, updater);
}
void CServer::shutdown(void)
{

3
Server.h
View File

@ -111,7 +111,8 @@ public:
virtual ISettingsReader* createDBMemSettingsReader(THREAD_ID tid, DATABASE_ID pIdentifier, const std::string &pTable, const std::string &pSQL = "");
virtual ISettingsReader* createDBMemSettingsReader(IDatabase *db, const std::string &pTable, const std::string &pSQL = "");
virtual ISettingsReader* createMemorySettingsReader(const std::string &pData);
virtual IPipeThrottler* createPipeThrottler(size_t bps, IPipeThrottlerUpdater* updater=NULL);
virtual IPipeThrottler* createPipeThrottler(size_t bps, bool percent_max);
virtual IPipeThrottler* createPipeThrottler(IPipeThrottlerUpdater* updater);
virtual bool openDatabase(std::string pFile, DATABASE_ID pIdentifier, const str_map& params = str_map(), std::string pEngine="sqlite");
virtual IDatabase* getDatabase(THREAD_ID tid, DATABASE_ID pIdentifier);

26
urbackupserver/ClientMain.cpp
View File

@ -2128,19 +2128,19 @@ int ClientMain::getNumberOfRunningFileBackups(void)
return running_file_backups;
}
IPipeThrottler *ClientMain::getThrottler(size_t speed_bps)
IPipeThrottler *ClientMain::getThrottler(int speed_bps)
{
IScopedLock lock(throttle_mutex);
if(client_throttler==NULL)
{
client_throttler=Server->createPipeThrottler(speed_bps,
new ThrottleUpdater(clientid,
client_throttler=Server->createPipeThrottler(new ThrottleUpdater(clientid,
internet_connection?ThrottleScope_Internet:ThrottleScope_Local));
}
else
{
client_throttler->changeThrottleLimit(speed_bps);
client_throttler->changeThrottleLimit(speed_bps>=0 ? speed_bps : -1*speed_bps,
speed_bps<0);
}
return client_throttler;
@ -2192,12 +2192,12 @@ IPipe *ClientMain::getClientCommandConnection(int timeoutms, std::string* client
if(server_settings!=NULL && ret!=NULL)
{
int internet_speed=server_settings->getInternetSpeed();
if(internet_speed>0)
if(internet_speed!=0)
{
ret->addThrottler(getThrottler(internet_speed));
}
int global_internet_speed=server_settings->getGlobalInternetSpeed();
if(global_internet_speed>0)
if(global_internet_speed!=0)
{
ret->addThrottler(BackupServer::getGlobalInternetThrottler(global_internet_speed));
}
@ -2242,12 +2242,12 @@ _u32 ClientMain::getClientFilesrvConnection(FileClient *fc, ServerSettings* serv
if(server_settings!=NULL)
{
int internet_speed=server_settings->getInternetSpeed();
if(internet_speed>0)
if(internet_speed!=0)
{
fc->addThrottler(getThrottler(internet_speed));
}
int global_internet_speed=server_settings->getGlobalInternetSpeed();
if(global_internet_speed>0)
if(global_internet_speed!=0)
{
fc->addThrottler(BackupServer::getGlobalInternetThrottler(global_internet_speed));
}
@ -2265,12 +2265,12 @@ _u32 ClientMain::getClientFilesrvConnection(FileClient *fc, ServerSettings* serv
if(server_settings!=NULL)
{
int local_speed=server_settings->getLocalSpeed();
if(local_speed>0)
if(local_speed!=0)
{
fc->addThrottler(getThrottler(local_speed));
}
int global_local_speed=server_settings->getGlobalLocalSpeed();
if(global_local_speed>0)
if(global_local_speed!=0)
{
fc->addThrottler(BackupServer::getGlobalLocalThrottler(global_local_speed));
}
@ -2329,7 +2329,7 @@ bool ClientMain::getClientChunkedFilesrvConnection(std::auto_ptr<FileClientChunk
{
speed=server_settings->getLocalSpeed();
}
if(speed>0)
if(speed!=0)
{
fc_chunked->addThrottler(getThrottler(speed));
}
@ -2337,7 +2337,7 @@ bool ClientMain::getClientChunkedFilesrvConnection(std::auto_ptr<FileClientChunk
if(internet_connection)
{
int global_speed=server_settings->getGlobalInternetSpeed();
if(global_speed>0)
if(global_speed!=0)
{
fc_chunked->addThrottler(BackupServer::getGlobalInternetThrottler(global_speed));
}
@ -2345,7 +2345,7 @@ bool ClientMain::getClientChunkedFilesrvConnection(std::auto_ptr<FileClientChunk
else
{
int global_speed=server_settings->getGlobalLocalSpeed();
if(global_speed>0)
if(global_speed!=0)
{
fc_chunked->addThrottler(BackupServer::getGlobalLocalThrottler(global_speed));
}

2
urbackupserver/ClientMain.h
View File

@ -250,7 +250,7 @@ private:
bool sendFile(IPipe *cc, IFile *f, int timeout);
bool isBackupsRunningOkay(bool file, bool incr=false);
bool updateCapabilities(void);
IPipeThrottler *getThrottler(size_t speed_bps);
IPipeThrottler *getThrottler(int speed_bps);
bool inBackupWindow(Backup* backup);
unsigned int exponentialBackoffTime(size_t count, unsigned int sleeptime, unsigned div);

23
urbackupserver/ThrottleUpdater.cpp
View File

@ -33,23 +33,36 @@ int64 ThrottleUpdater::getUpdateIntervalMs()
return 10*60*1000; //10 min
}
size_t ThrottleUpdater::getThrottleLimit()
size_t ThrottleUpdater::getThrottleLimit(bool& percent_max)
{
IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER);
ServerSettings server_settings(db, clientid);
int ret;
switch(throttle_scope)
{
case ThrottleScope_GlobalInternet:
return server_settings.getGlobalInternetSpeed();
ret = server_settings.getGlobalInternetSpeed();
case ThrottleScope_GlobalLocal:
return server_settings.getGlobalLocalSpeed();
ret = server_settings.getGlobalLocalSpeed();
case ThrottleScope_Internet:
return server_settings.getInternetSpeed();
ret = server_settings.getInternetSpeed();
case ThrottleScope_Local:
return server_settings.getLocalSpeed();
ret = server_settings.getLocalSpeed();
default:
percent_max = false;
return std::string::npos;
}
if (ret < 0)
{
percent_max = true;
return ret*-1;
}
else
{
percent_max = false;
return ret;
}
}

2
urbackupserver/ThrottleUpdater.h
View File

@ -16,7 +16,7 @@ public:
virtual int64 getUpdateIntervalMs();
virtual size_t getThrottleLimit();
virtual size_t getThrottleLimit(bool& percent_max);
private:
int clientid;

21
urbackupserver/dllmain.cpp
View File

@ -1832,6 +1832,18 @@ bool upgrade48_49()
return b;
}
bool upgrade49_50()
{
IDatabase *db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER);
bool b = true;
b &= db->Write("UPDATE settings_db.settings SET value='0' WHERE value='-1' "
"AND (key='local_speed' OR key='internet_speed' OR key='global_local_speed' OR key='global_internet_speed')");
return b;
}
void upgrade(void)
{
@ -1854,7 +1866,7 @@ void upgrade(void)
int ver=watoi(res_v[0]["tvalue"]);
int old_v;
int max_v=49;
int max_v=50;
{
IScopedLock lock(startup_status.mutex);
startup_status.target_db_version=max_v;
@ -2134,6 +2146,13 @@ void upgrade(void)
}
++ver;
break;
case 49:
if (!upgrade49_50())
{
has_error = true;
}
++ver;
break;
default:
break;
}

14
urbackupserver/server.cpp
View File

@ -582,7 +582,7 @@ void BackupServer::removeAllClients(void)
}
}
IPipeThrottler *BackupServer::getGlobalInternetThrottler(size_t speed_bps)
IPipeThrottler *BackupServer::getGlobalInternetThrottler(int speed_bps)
{
IScopedLock lock(throttle_mutex);
@ -591,17 +591,18 @@ IPipeThrottler *BackupServer::getGlobalInternetThrottler(size_t speed_bps)
if(global_internet_throttler==NULL)
{
global_internet_throttler=Server->createPipeThrottler(speed_bps,
global_internet_throttler=Server->createPipeThrottler(
new ThrottleUpdater(-1, ThrottleScope_GlobalInternet));
}
else
{
global_internet_throttler->changeThrottleLimit(speed_bps);
global_internet_throttler->changeThrottleLimit(speed_bps>=0 ? speed_bps : -1*speed_bps,
speed_bps<0);
}
return global_internet_throttler;
}
IPipeThrottler *BackupServer::getGlobalLocalThrottler(size_t speed_bps)
IPipeThrottler *BackupServer::getGlobalLocalThrottler(int speed_bps)
{
IScopedLock lock(throttle_mutex);
@ -610,12 +611,13 @@ IPipeThrottler *BackupServer::getGlobalLocalThrottler(size_t speed_bps)
if(global_local_throttler==NULL)
{
global_local_throttler=Server->createPipeThrottler(speed_bps,
global_local_throttler=Server->createPipeThrottler(
new ThrottleUpdater(-1, ThrottleScope_GlobalLocal));
}
else
{
global_local_throttler->changeThrottleLimit(speed_bps);
global_local_throttler->changeThrottleLimit(speed_bps >= 0 ? speed_bps : -1 * speed_bps,
speed_bps<0);
}
return global_local_throttler;
}

4
urbackupserver/server.h
View File

@ -28,8 +28,8 @@ public:
void operator()(void);
static IPipeThrottler *getGlobalInternetThrottler(size_t speed_bps);
static IPipeThrottler *getGlobalLocalThrottler(size_t speed_bps);
static IPipeThrottler *getGlobalInternetThrottler(int speed_bps);
static IPipeThrottler *getGlobalLocalThrottler(int speed_bps);
static void cleanupThrottlers(void);

80
urbackupserver/www/js/urbackup.js
View File

@ -389,7 +389,7 @@ function build_main_nav()
})
}
function multiplyTimeSpan(ts, m)
function multiplyTimeSpan(ts, m, allow_percent)
{
ts = unescapeHTML(ts);
var timespans = ts.split(";");
@ -404,13 +404,51 @@ function multiplyTimeSpan(ts, m)
var idx = timespans[i].indexOf("@");
if(idx!=-1)
{
var d=parseFloat(timespans[i].substr(0, idx));
ret+=d*m + timespans[i].substr(idx, timespans[i].length - idx);
var idx_pc = timespans[i].indexOf("%");
if(idx_pc!=-1 && allow_pc )
{
var d=parseFloat(timespans[i].substr(0, idx_pc));
ret+=d*-1 + timespans[i].substr(idx, timespans[i].length - idx);
}
else
{
var d=parseFloat(timespans[i].substr(0, idx));
if(d<0 && m<1 && allow_percent)
{
ret+=(d*-1)+"%";
}
else
{
ret+=d*m;
}
ret+=timespans[i].substr(idx, timespans[i].length - idx);
}
}
else
{
d=parseFloat(timespans[i]);
ret+=d*m;
var idx_pc = timespans[i].indexOf("%");
if(idx_pc!=-1 && allow_pc )
{
var d=parseFloat(timespans[i].substr(0, idx_pc));
ret+=d*-1;
}
else
{
var d=parseFloat(timespans[i]);
if(d<0 && m<1 && allow_percent)
{
ret+=(d*-1)+"%";
}
else
{
ret+=d*m;
}
ret+=d*m;
}
}
}
@ -464,10 +502,10 @@ function getPar(p)
if(p=="update_freq_full" || p=="update_freq_image_full" || p=="update_freq_image_incr")
{ val=multiplyTimeSpan(val, 60.0*60.0*24.0); if(obj.disabled) val=makeTimeSpanNegative(val); }
if(p=="startup_backup_delay") val*=60;
if(p=="local_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, (1024*1024)/8); }
if(p=="internet_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, 1024/8); }
if(p=="global_local_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, (1024*1024)/8); }
if(p=="global_internet_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, 1024/8); }
if(p=="local_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, (1024*1024)/8, true); }
if(p=="internet_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, 1024/8, true); }
if(p=="global_local_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, (1024*1024)/8, true); }
if(p=="global_internet_speed") { if(val=="-" || val=="") val=-1; else val=multiplyTimeSpan(val, 1024/8, true); }
if(p=="update_stats_cachesize") val=Math.round(val*1024);
return "&"+p+"="+encodeURIComponent(val+"");
@ -2614,15 +2652,15 @@ function show_settings2(data)
data.settings.update_freq_image_full=multiplyTimeSpan(data.settings.update_freq_image_full, 1/(60.0*60.0*24.0));
data.settings.startup_backup_delay/=60;
if(data.settings.local_speed=="-1") data.settings.local_speed="-";
else data.settings.local_speed=multiplyTimeSpan(data.settings.local_speed, 1/((1024*1024)/8));
if(data.settings.internet_speed=="-1") data.settings.internet_speed="-";
else data.settings.internet_speed=multiplyTimeSpan(data.settings.internet_speed, 1/(1024/8));
if(data.settings.local_speed=="0") data.settings.local_speed="-";
else data.settings.local_speed=multiplyTimeSpan(data.settings.local_speed, 1/((1024*1024)/8), true);
if(data.settings.internet_speed=="0") data.settings.internet_speed="-";
else data.settings.internet_speed=multiplyTimeSpan(data.settings.internet_speed, 1/(1024/8), true);
if(data.settings.global_local_speed=="-1") data.settings.global_local_speed="-";
else data.settings.global_local_speed=multiplyTimeSpan(data.settings.global_local_speed, 1/((1024*1024)/8));
if(data.settings.global_internet_speed=="-1") data.settings.global_internet_speed="-";
else data.settings.global_internet_speed=multiplyTimeSpan(data.settings.global_internet_speed, 1/(1024/8));
if(data.settings.global_local_speed=="0") data.settings.global_local_speed="-";
else data.settings.global_local_speed=multiplyTimeSpan(data.settings.global_local_speed, 1/((1024*1024)/8), true);
if(data.settings.global_internet_speed=="0") data.settings.global_internet_speed="-";
else data.settings.global_internet_speed=multiplyTimeSpan(data.settings.global_internet_speed, 1/(1024/8), true);
data.settings.file_hash_collect_cachesize/=1024;
data.settings.update_stats_cachesize/=1024;
@ -2747,10 +2785,10 @@ function show_settings2(data)
data.settings.update_freq_image_full=multiplyTimeSpan(data.settings.update_freq_image_full, 1/(60.0*60.0*24.0));
data.settings.startup_backup_delay/=60;
if(data.settings.local_speed=="-1") data.settings.local_speed="-";
else data.settings.local_speed=multiplyTimeSpan(data.settings.local_speed, 1/((1024*1024)/8));
if(data.settings.internet_speed=="-1") data.settings.internet_speed="-";
else data.settings.internet_speed=multiplyTimeSpan(data.settings.internet_speed, 1/(1024/8));
if(data.settings.local_speed=="0") data.settings.local_speed="-";
else data.settings.local_speed=multiplyTimeSpan(data.settings.local_speed, 1/((1024*1024)/8), true);
if(data.settings.internet_speed=="0") data.settings.internet_speed="-";
else data.settings.internet_speed=multiplyTimeSpan(data.settings.internet_speed, 1/(1024/8), true);
data.settings.file_hash_collect_cachesize/=1024;