mirrors/urbackup_backend
2
0
Fork 0
mirror of https://github.com/uroni/urbackup_backend.git synced 2025-10-26 11:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Access files directly from "tray icon" without logging in. Works by saving the user access rights for each backup

Browse Source
This commit is contained in:
Martin 2014-06-25 21:23:47 +02:00
parent 1fc820978f
commit 9d453785cf
63 changed files with 2592 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -209,3 +209,5 @@ urbackup/server_ident.priv
urbackup/server_ident.pub
urbackupserver/www/polib.pyc
urbackup/server_version_info.properties
access_keys.properties
tokens/*

125
FileSettingsReader.cpp
View File

@ -20,10 +20,35 @@
#include "stringtools.h"
#include "Server.h"
#include <iostream>
#include <memory>
#include "Interface/File.h"
std::map<std::string, SCachedSettings*>* CFileSettingsReader::settings=new std::map<std::string, SCachedSettings*>;
IMutex* CFileSettingsReader::settings_mutex=NULL;
namespace
{
std::string readFile(const std::wstring& fn)
{
std::auto_ptr<IFile> file(Server->openFile(fn));
if(file.get()==NULL)
{
return std::string();
}
std::string ret;
ret.resize(file->Size());
size_t read=0;
while(read<ret.size())
{
read+=file->Read(&ret[read], static_cast<_u32>(ret.size()-read));
}
return ret;
}
}
CFileSettingsReader::CFileSettingsReader(std::string pFile)
{
std::map<std::string, SCachedSettings*>::iterator iter;
@ -35,42 +60,7 @@ CFileSettingsReader::CFileSettingsReader(std::string pFile)
{
std::string fdata=getFile(pFile);
std::vector<SSetting> mSettings;
int num_lines=linecount(fdata);
for(int i=0;i<num_lines;++i)
{
std::string line=getline(i,fdata);
if(line.size()<2 || line[0]=='#' )
continue;
SSetting setting;
setting.key=getuntil("=",line);
if(setting.key=="")
setting.value=line;
else
{
line.erase(0,setting.key.size()+1);
setting.value=line;
}
mSettings.push_back(setting);
}
cached_settings=new SCachedSettings;
cached_settings->smutex=Server->createMutex();
for(size_t i=0;i<mSettings.size();++i)
{
cached_settings->mSettingsMap[Server->ConvertToUnicode(mSettings[i].key)]=Server->ConvertToUnicode(mSettings[i].value);
}
cached_settings->refcount=1;
cached_settings->key=pFile;
IScopedLock lock(settings_mutex);
settings->insert(std::pair<std::string, SCachedSettings*>(pFile, cached_settings) );
read(fdata, pFile);
}
else
{
@ -81,6 +71,27 @@ CFileSettingsReader::CFileSettingsReader(std::string pFile)
}
CFileSettingsReader::CFileSettingsReader( std::wstring pFile )
{
std::map<std::string, SCachedSettings*>::iterator iter;
{
IScopedLock lock(settings_mutex);
iter=settings->find(Server->ConvertToUTF8(pFile));
}
if( iter==settings->end() )
{
std::string fdata=readFile(pFile);
read(fdata, Server->ConvertToUTF8(pFile));
}
else
{
IScopedLock lock(settings_mutex);
cached_settings=iter->second;
++cached_settings->refcount;
}
}
CFileSettingsReader::~CFileSettingsReader()
{
IScopedLock lock(settings_mutex);
@ -155,4 +166,44 @@ std::vector<std::wstring> CFileSettingsReader::getKeys()
ret.push_back(i->first);
}
return ret;
}
}
void CFileSettingsReader::read( const std::string& fdata, const std::string& pFile )
{
std::vector<SSetting> mSettings;
int num_lines=linecount(fdata);
for(int i=0;i<num_lines;++i)
{
std::string line=getline(i,fdata);
if(line.size()<2 || line[0]=='#' )
continue;
SSetting setting;
setting.key=getuntil("=",line);
if(setting.key=="")
setting.value=line;
else
{
line.erase(0,setting.key.size()+1);
setting.value=line;
}
mSettings.push_back(setting);
}
cached_settings=new SCachedSettings;
cached_settings->smutex=Server->createMutex();
for(size_t i=0;i<mSettings.size();++i)
{
cached_settings->mSettingsMap[Server->ConvertToUnicode(mSettings[i].key)]=Server->ConvertToUnicode(mSettings[i].value);
}
cached_settings->refcount=1;
cached_settings->key=pFile;
IScopedLock lock(settings_mutex);
settings->insert(std::pair<std::string, SCachedSettings*>(pFile, cached_settings) );
}

3
FileSettingsReader.h
View File

@ -22,6 +22,7 @@ class CFileSettingsReader : public CSettingsReader
{
public:
CFileSettingsReader(std::string pFile);
CFileSettingsReader(std::wstring pFile);
~CFileSettingsReader();
virtual bool getValue(std::string key, std::string *value);
@ -33,6 +34,8 @@ public:
virtual std::vector<std::wstring> getKeys();
private:
void read(const std::string& fdata, const std::string& pFile);
static std::map<std::string, SCachedSettings*> *settings;
static IMutex *settings_mutex;

8
Interface/Server.h
View File

@ -94,7 +94,8 @@ public:
virtual void createThread(IThread *thread)=0;
virtual IPipe *createMemoryPipe(void)=0;
virtual IThreadPool *getThreadPool(void)=0;
virtual ISettingsReader* createFileSettingsReader(std::string pFile)=0;
virtual ISettingsReader* createFileSettingsReader(const std::string& pFile)=0;
virtual ISettingsReader* createFileSettingsReader(const std::wstring& pFile)=0;
virtual ISettingsReader* createDBSettingsReader(THREAD_ID tid, DATABASE_ID pIdentifier, const std::string &pTable, const std::string &pSQL="")=0;
virtual ISettingsReader* createDBSettingsReader(IDatabase *db, const std::string &pTable, const std::string &pSQL="")=0;
virtual ISettingsReader* createMemorySettingsReader(const std::string &pData)=0;
@ -143,6 +144,8 @@ public:
virtual IFile* openMemoryFile(void)=0;
virtual bool deleteFile(std::string pFilename)=0;
virtual bool deleteFile(std::wstring pFilename)=0;
virtual bool fileExists(std::string pFilename)=0;
virtual bool fileExists(std::wstring pFilename)=0;
virtual POSTFILE_KEY getPostFileKey()=0;
virtual void addPostFile(POSTFILE_KEY pfkey, const std::string &name, const SPostfile &pf)=0;
@ -169,7 +172,8 @@ public:
virtual unsigned int getSecureRandomNumber(void)=0;
virtual std::vector<unsigned int> getSecureRandomNumbers(size_t n)=0;
virtual void secureRandomFill(char *buf, size_t blen)=0;
virtual std::string secureRandomString(size_t len)=0;
static const size_t FAIL_DATABASE_CORRUPTED=1;
static const size_t FAIL_DATABASE_IOERR=2;

36
Server.cpp
View File

@ -1302,7 +1302,12 @@ IThreadPool *CServer::getThreadPool(void)
return threadpool;
}
ISettingsReader* CServer::createFileSettingsReader(std::string pFile)
ISettingsReader* CServer::createFileSettingsReader(const std::string& pFile)
{
return new CFileSettingsReader(pFile);
}
ISettingsReader* CServer::createFileSettingsReader(const std::wstring& pFile)
{
return new CFileSettingsReader(pFile);
}
@ -1400,6 +1405,25 @@ bool CServer::deleteFile(std::wstring pFilename)
return DeleteFileInt(pFilename);
}
bool CServer::fileExists(std::string pFilename)
{
return FileExists(pFilename);
}
bool CServer::fileExists(std::wstring pFilename)
{
#ifndef WIN32
return FileExists(ConvertToUTF8(pFilename));
#else
fstream in(pFilename.c_str(), ios::in);
if( in.is_open()==false )
return false;
in.close();
return true;
#endif
}
std::string CServer::ConvertToUTF8(const std::wstring &input)
{
std::string ret;
@ -1800,6 +1824,16 @@ void CServer::secureRandomFill(char *buf, size_t blen)
#endif
}
std::string CServer::secureRandomString(size_t len)
{
std::string rchars="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
std::string key;
std::vector<unsigned int> rnd_n=Server->getSecureRandomNumbers(len);
for(size_t j=0;j<len;++j)
key+=rchars[rnd_n[j]%rchars.size()];
return key;
}
void CServer::setLogCircularBufferSize(size_t size)
{
IScopedLock lock(log_mutex);

6
Server.h
View File

@ -88,7 +88,8 @@ public:
virtual IPipe *createMemoryPipe(void);
virtual void createThread(IThread *thread);
virtual IThreadPool *getThreadPool(void);
virtual ISettingsReader* createFileSettingsReader(std::string pFile);
virtual ISettingsReader* createFileSettingsReader(const std::string& pFile);
virtual ISettingsReader* createFileSettingsReader(const std::wstring& pFile);
virtual ISettingsReader* createDBSettingsReader(THREAD_ID tid, DATABASE_ID pIdentifier, const std::string &pTable, const std::string &pSQL="");
virtual ISettingsReader* createDBSettingsReader(IDatabase *db, const std::string &pTable, const std::string &pSQL="");
virtual ISettingsReader* createMemorySettingsReader(const std::string &pData);
@ -138,6 +139,8 @@ public:
virtual IFile* openMemoryFile(void);
virtual bool deleteFile(std::string pFilename);
virtual bool deleteFile(std::wstring pFilename);
virtual bool fileExists(std::string pFilename);
virtual bool fileExists(std::wstring pFilename);
virtual POSTFILE_KEY getPostFileKey();
virtual void addPostFile(POSTFILE_KEY pfkey, const std::string &name, const SPostfile &pf);
@ -171,6 +174,7 @@ public:
virtual unsigned int getSecureRandomNumber(void);
virtual std::vector<unsigned int> getSecureRandomNumbers(size_t n);
virtual void secureRandomFill(char *buf, size_t blen);
virtual std::string secureRandomString(size_t len);
virtual void setFailBit(size_t failbit);
virtual void clearFailBit(size_t failbit);

60
cryptoplugin/CryptoFactory.cpp
View File

@ -207,3 +207,63 @@ bool CryptoFactory::verifyData( const std::string &pubkey, const std::string &da
return false;
}
std::string CryptoFactory::encryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations/*=20000*/ )
{
const size_t iv_size=CryptoPP::AES::BLOCKSIZE;
std::string ciphertext;
ciphertext.resize(iv_size);
Server->secureRandomFill(&ciphertext[0], iv_size);
CryptoPP::SecByteBlock key;
key.resize(CryptoPP::SHA256::DIGESTSIZE);
CryptoPP::PKCS5_PBKDF2_HMAC<CryptoPP::SHA512> gen;
gen.DeriveKey(key.BytePtr(), CryptoPP::SHA256::DIGESTSIZE, 0, reinterpret_cast<const byte*>(password.c_str()), password.size(),
reinterpret_cast<const byte*>(ciphertext.data()), iv_size, static_cast<unsigned int>(iterations), 0);
CryptoPP::EAX<CryptoPP::AES>::Encryption enc;
enc.SetKeyWithIV(key.BytePtr(), CryptoPP::AES::BLOCKSIZE, reinterpret_cast<const byte*>(ciphertext.data()), iv_size);
CryptoPP::ArraySource( reinterpret_cast<const byte*>(data.data()), data.size(), true,
new CryptoPP::AuthenticatedEncryptionFilter( enc,
new CryptoPP::StringSink( ciphertext )
) );
return ciphertext;
}
std::string CryptoFactory::decryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations)
{
const size_t iv_size=CryptoPP::AES::BLOCKSIZE;
if(data.size()<iv_size)
{
return std::string();
}
CryptoPP::SecByteBlock key;
key.resize(CryptoPP::SHA256::DIGESTSIZE);
CryptoPP::PKCS5_PBKDF2_HMAC<CryptoPP::SHA512> gen;
gen.DeriveKey(key.BytePtr(), CryptoPP::SHA256::DIGESTSIZE, 0, reinterpret_cast<const byte*>(password.c_str()), password.size(),
reinterpret_cast<const byte*>(data.data()), iv_size, static_cast<unsigned int>(iterations), 0);
CryptoPP::EAX<CryptoPP::AES>::Decryption dec;
dec.SetKeyWithIV(key.BytePtr(), CryptoPP::AES::BLOCKSIZE, reinterpret_cast<const byte*>(data.data()), iv_size);
try
{
std::string ret;
CryptoPP::ArraySource( reinterpret_cast<const byte*>(data.data()+iv_size), data.size()-iv_size, true,
new CryptoPP::AuthenticatedDecryptionFilter( dec,
new CryptoPP::StringSink( ret ) ) );
return ret;
}
catch(const CryptoPP::HashVerificationFilter::HashVerificationFailed&)
{
return std::string();
}
}

2
cryptoplugin/CryptoFactory.h
View File

@ -16,4 +16,6 @@ public:
virtual bool verifyData(const std::string &pubkey, const std::string &data, const std::string &signature);
virtual std::string generatePasswordHash(const std::string &password, const std::string &salt, size_t iterations=10000);
virtual std::string generateBinaryPasswordHash(const std::string &password, const std::string &salt, size_t iterations=10000);
virtual std::string encryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations=20000);
virtual std::string decryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations=20000);
};

2
cryptoplugin/ICryptoFactory.h
View File

@ -21,4 +21,6 @@ public:
virtual bool verifyData(const std::string &pubkey, const std::string &data, const std::string &signature)=0;
virtual std::string generatePasswordHash(const std::string &password, const std::string &salt, size_t iterations=10000)=0;
virtual std::string generateBinaryPasswordHash(const std::string &password, const std::string &salt, size_t iterations=10000)=0;
virtual std::string encryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations=20000)=0;
virtual std::string decryptAuthenticatedAES(const std::string& data, const std::string &password, size_t iterations=20000)=0;
};

3
cryptoplugin/cryptopp_inc.h
View File

@ -8,6 +8,7 @@
#include <files.h>
#include <pwdbased.h>
#include <hex.h>
#include <eax.h>
#else
#include "config.h"
#define CRYPTOPP_INCLUDE_AES <CRYPTOPP_INCLUDE_PREFIX/aes.h>
@ -19,6 +20,7 @@
#define CRYPTOPP_INCLUDE_FILES <CRYPTOPP_INCLUDE_PREFIX/files.h>
#define CRYPTOPP_INCLUDE_PWDBASED <CRYPTOPP_INCLUDE_PREFIX/pwdbased.h>
#define CRYPTOPP_INCLUDE_HEX <CRYPTOPP_INCLUDE_PREFIX/hex.h>
#define CRYPTOPP_INCLUDE_EAX <CRYPTOPP_INCLUDE_PREFIX/eax.h>
#include CRYPTOPP_INCLUDE_AES
#include CRYPTOPP_INCLUDE_SHA
@ -29,4 +31,5 @@
#include CRYPTOPP_INCLUDE_FILES
#include CRYPTOPP_INCLUDE_PWDBASED
#include CRYPTOPP_INCLUDE_HEX
#include CRYPTOPP_INCLUDE_EAX
#endif

5
fileservplugin/chunk_settings.h
View File

@ -9,4 +9,9 @@ const unsigned int c_chunk_padding=1+sizeof(_i64)+sizeof(_u32);
const unsigned int small_hash_size=4;
const unsigned int big_hash_size=16;
const unsigned int chunkhash_file_off=sizeof(_i64);
const unsigned int chunkhash_single_size=big_hash_size+small_hash_size*(c_checkpoint_dist/c_small_hash_dist);
const unsigned int c_reconnection_tries=30;
#endif //CHUNK_SETTINGS_H

1
stringtools.cpp
View File

@ -40,7 +40,6 @@ typedef int s32;
typedef unsigned int u32;
typedef float f32;
//--------------------------------------------------------------------
/**
* liefert einen teil des strings nach dem gelieferten teilstring

95
urbackupclient/ClientService.cpp
View File

@ -34,10 +34,13 @@
#include "../urbackupcommon/settings.h"
#include "ImageThread.h"
#include "InternetClient.h"
#include "../urbackupcommon/settingslist.h"
#include <memory.h>
#include <stdlib.h>
#include <limits.h>
#include <memory>
#include <algorithm>
#ifndef _WIN32
#define _atoi64 atoll
@ -900,6 +903,10 @@ void ClientConnector::ReceivePackets(void)
{
CMD_RESTORE_DOWNLOADPROGRESS(cmd); continue;
}
else if( cmd=="GET ACCESS PARAMETERS")
{
CMD_GET_ACCESS_PARAMS(params); continue;
}
}
if(is_channel) //Channel commands from server
{
@ -1133,8 +1140,8 @@ std::vector<std::wstring> getSettingsList(void);
void ClientConnector::updateSettings(const std::string &pData)
{
ISettingsReader *curr_settings=Server->createFileSettingsReader("urbackup/data/settings.cfg");
ISettingsReader *new_settings=Server->createMemorySettingsReader(pData);
std::auto_ptr<ISettingsReader> curr_settings(Server->createFileSettingsReader("urbackup/data/settings.cfg"));
std::auto_ptr<ISettingsReader> new_settings(Server->createMemorySettingsReader(pData));
std::vector<std::wstring> settings_names=getSettingsList();
settings_names.push_back(L"client_set_settings");
@ -1153,13 +1160,19 @@ void ClientConnector::updateSettings(const std::string &pData)
client_set_settings=true;
}
}
for(size_t i=0;i<settings_names.size();++i)
{
std::wstring key=settings_names[i];
std::wstring v;
std::wstring def_v;
curr_settings->getValue(key+L"_def", def_v);
if(!curr_settings->getValue(key, &v) )
{
std::wstring nv;
std::wstring new_key;
if(new_settings->getValue(key, &nv) )
{
new_settings_str+=key+L"="+nv+L"\n";
@ -1168,8 +1181,11 @@ void ClientConnector::updateSettings(const std::string &pData)
if(new_settings->getValue(key+L"_def", &nv) )
{
new_settings_str+=key+L"_def="+nv+L"\n";
mod=true;
}
if(nv!=def_v)
{
mod=true;
}
}
}
else
{
@ -1183,7 +1199,10 @@ void ClientConnector::updateSettings(const std::string &pData)
new_settings->getValue(key+L"_def", &nv) ) )
{
new_settings_str+=key+L"="+nv+L"\n";
mod=true;
if(nv!=v)
{
mod=true;
}
}
else
{
@ -1203,7 +1222,10 @@ void ClientConnector::updateSettings(const std::string &pData)
else
{
new_settings_str+=key+L"="+nv+L"\n";
mod=true;
if(v!=nv)
{
mod=true;
}
}
}
else if(key==L"internet_server" && !v.empty()
@ -1215,7 +1237,10 @@ void ClientConnector::updateSettings(const std::string &pData)
if(new_settings->getValue(key+L"_def", &nv) )
{
new_settings_str+=key+L"_def="+nv+L"\n";
mod=true;
if(nv!=def_v)
{
mod=true;
}
}
}
}
@ -1254,23 +1279,55 @@ void ClientConnector::updateSettings(const std::string &pData)
}
}
Server->destroy(curr_settings);
Server->destroy(new_settings);
if(mod)
{
IFile *sf=Server->openFile("urbackup/data/settings.cfg", MODE_WRITE );
if(sf==NULL)
{
Server->Log("Error opening settings file!", LL_ERROR);
return;
}
sf->Write(Server->ConvertToUTF8(new_settings_str));
Server->destroy(sf);
writestring(Server->ConvertToUTF8(new_settings_str), "urbackup/data/settings.cfg");
InternetClient::updateSettings();
}
std::auto_ptr<ISettingsReader> curr_server_settings(Server->createFileSettingsReader("urbackup/data/settings_"+server_token+".cfg"));
std::vector<std::wstring> global_settings = getGlobalizedSettingsList();
std::wstring new_token_settings=L"";
bool mod_server_settings=false;
for(size_t i=0;i<global_settings.size();++i)
{
std::wstring key=global_settings[i];
std::wstring v;
bool curr_v=curr_server_settings->getValue(key, &v);
std::wstring nv;
bool new_v=new_settings->getValue(key, &nv);
if(!curr_v && new_v)
{
new_token_settings+=key+L"="+nv;
mod_server_settings=true;
}
else if(curr_v)
{
if(new_v)
{
new_token_settings+=key+L"="+nv;
if(nv!=v)
{
mod_server_settings=true;
}
}
else
{
new_token_settings+=key+L"="+v;
}
}
}
if(mod_server_settings)
{
writestring(Server->ConvertToUTF8(new_settings_str), "urbackup/data/settings_"+server_token+".cfg");
}
}
void ClientConnector::replaceSettings(const std::string &pData)

1
urbackupclient/ClientService.h
View File

@ -175,6 +175,7 @@ private:
void CMD_NEW_SERVER(str_map &params);
void CMD_ENABLE_END_TO_END_FILE_BACKUP_VERIFICATION(const std::string &cmd);
void CMD_GET_VSSLOG(const std::string &cmd);
void CMD_GET_ACCESS_PARAMS(str_map &params);
IPipe *pipe;
IPipe *mempipe;

84
urbackupclient/ClientServiceCMD.cpp
View File

@ -1,4 +1,5 @@
#include "../Interface/Server.h"
#include "../Interface/SettingsReader.h"
#include "ClientService.h"
#include "InternetClient.h"
#include "ServerIdentityMgr.h"
@ -22,6 +23,7 @@ std::wstring getSysVolume(std::wstring &mpath){ return L""; }
#include <stdlib.h>
#include <limits.h>
#ifndef _WIN32
#define _atoi64 atoll
@ -31,19 +33,6 @@ extern std::string time_format_str;
extern ICryptoFactory *crypto_fak;
namespace
{
std::string randomChallenge(size_t len)
{
std::string rchars="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
std::string key;
std::vector<unsigned int> rnd_n=Server->getSecureRandomNumbers(len);
for(size_t j=0;j<len;++j)
key+=rchars[rnd_n[j]%rchars.size()];
return key;
}
}
void ClientConnector::CMD_ADD_IDENTITY(const std::string &identity, const std::string &cmd, bool ident_ok)
{
if(identity.empty())
@ -104,7 +93,7 @@ void ClientConnector::CMD_GET_CHALLENGE(const std::string &identity)
}
IScopedLock lock(ident_mutex);
std::string challenge = randomChallenge(30)+"-"+nconvert(Server->getTimeSeconds())+"-"+nconvert(Server->getTimeMS());
std::string challenge = Server->secureRandomString(30)+"-"+nconvert(Server->getTimeSeconds())+"-"+nconvert(Server->getTimeMS());
challenges[identity]=challenge;
tcpstack.Send(pipe, challenge);
@ -1372,4 +1361,71 @@ void ClientConnector::CMD_GET_VSSLOG(const std::string &cmd)
localpipe->Read(&ret, 8000);
tcpstack.Send(pipe, ret);
localpipe->Write("exit");
}
void ClientConnector::CMD_GET_ACCESS_PARAMS(str_map &params)
{
if(crypto_fak==NULL)
{
Server->Log("No cryptoplugin present. Action not possible.", LL_ERROR);
tcpstack.Send(pipe, "");
return;
}
std::wstring tokens=params[L"tokens"];
std::auto_ptr<ISettingsReader> settings(
Server->createFileSettingsReader("urbackup/data/settings.cfg"));
std::string computername;
if( (!settings->getValue("computername", &computername)
&& !settings->getValue("computername_def", &computername) )
|| computername.empty())
{
computername=Server->ConvertToUTF8(IndexThread::getFileSrv()->getServerName());
}
std::string server_url;
if( (!settings->getValue("server_url", &server_url)
&& !settings->getValue("server_url_def", &server_url) )
|| server_url.empty())
{
Server->Log("Server url empty", LL_ERROR);
tcpstack.Send(pipe, "");
return;
}
std::auto_ptr<ISettingsReader> access_keys(
Server->createFileSettingsReader("access_keys.properties"));
std::vector<std::wstring> server_token_keys = access_keys->getKeys();
if(server_token_keys.empty())
{
Server->Log("No access key present", LL_ERROR);
tcpstack.Send(pipe, "");
return;
}
if(server_url[server_url.size()-1]!='/')
{
server_url+="/";
}
std::string ret = server_url+"#computername="+computername;
for(size_t i=0;i<server_token_keys.size();++i)
{
std::wstring server_key;
if(access_keys->getValue(server_token_keys[i],
&server_key) && !server_key.empty())
{
ret += "&tokens"+nconvert(i)+"="+base64_encode_dash(
crypto_fak->encryptAuthenticatedAES(Server->ConvertToUTF8(tokens),
Server->ConvertToUTF8(server_key) ) );
}
}
tcpstack.Send(pipe, ret);
}

220
urbackupclient/client.cpp
View File

@ -34,6 +34,8 @@
#include <algorithm>
#include <fstream>
#include <stdlib.h>
#include "win_tokens.h"
#include "file_permissions.h"
//For truncating files
#ifdef _WIN32
@ -619,6 +621,18 @@ void IndexThread::indexDirs(void)
}
updateDirs();
writeTokens();
bool tokens_changed=false;
if(cd->getMiscValue("has_new_token")==L"true")
{
VSSLog("File access tokens changed (e.g. new user). Not using db...", LL_INFO);
tokens_changed=true;
}
token_cache.reset();
#ifdef _WIN32
//Invalidate cache
DirectoryWatcherThread::freeze();
@ -742,11 +756,15 @@ void IndexThread::indexDirs(void)
index_c_db=0;
index_c_fs=0;
index_c_db_update=0;
outfile << "d\"" << escapeListName(Server->ConvertToUTF8(backup_dirs[i].tname)) << "\"\n";
SFile dirInfo = getFileMetadata(os_file_prefix(mod_path));
std::string dir_permissions;
readPermissions(mod_path, dir_permissions);
writeDir(outfile, backup_dirs[i].tname, dirInfo.created, dirInfo.last_modified, dir_permissions);
//db->Write("BEGIN IMMEDIATE;");
last_transaction_start=Server->getTimeMS();
index_root_path=mod_path;
initialCheck( backup_dirs[i].path, mod_path, backup_dirs[i].tname, outfile, true, backup_dirs[i].optional, !patterns_changed);
initialCheck( backup_dirs[i].path, mod_path, backup_dirs[i].tname, outfile, true,
backup_dirs[i].optional, !(patterns_changed || tokens_changed) );
cd->copyFromTmpFiles();
commitModifyFilesBuffer();
@ -829,6 +847,10 @@ void IndexThread::indexDirs(void)
{
readPatterns(patterns_changed, true);
}
if(tokens_changed)
{
cd->updateMiscValue("has_new_token", L"false");
}
share_dirs();
}
@ -906,23 +928,20 @@ bool IndexThread::initialCheck(const std::wstring &orig_dir, const std::wstring
continue;
}
has_include=true;
outfile << "f\"" << escapeListName(Server->ConvertToUTF8(files[i].name)) << "\" " << files[i].size << " " << files[i].last_modified;
outfile << "f\"" << escapeListName(Server->ConvertToUTF8(files[i].name)) << "\" " << files[i].size << " " << files[i].last_modified << "#";
if(end_to_end_file_backup_verification_enabled || calculate_filehashes_on_client)
outfile << "rpb=" << base64_encode_dash(files[i].file_permission_bits)
<< "&mod=" << nconvert(files[i].last_modified_orig)
<< "&creat=" << nconvert(files[i].created);
if(calculate_filehashes_on_client)
{
outfile << "#";
if(calculate_filehashes_on_client)
{
outfile << "sha512=" << base64_encode_dash(files[i].hash);
}
outfile << "&sha512=" << base64_encode_dash(files[i].hash);
}
if(end_to_end_file_backup_verification_enabled)
{
if(calculate_filehashes_on_client) outfile << "&";
outfile << "sha256=" << getSHA256(dir+os_file_sep()+files[i].name);
}
if(end_to_end_file_backup_verification_enabled)
{
outfile << "&sha256=" << getSHA256(dir+os_file_sep()+files[i].name);
}
outfile << "\n";
@ -948,8 +967,11 @@ bool IndexThread::initialCheck(const std::wstring &orig_dir, const std::wstring
if( curr_included || !adding_worthless1 || !adding_worthless2 )
{
std::streampos pos=outfile.tellp();
outfile << "d\"" << escapeListName(Server->ConvertToUTF8(files[i].name)) << "\"\n";
bool b=initialCheck(orig_dir+os_file_sep()+files[i].name, dir+os_file_sep()+files[i].name, named_path+os_file_sep()+files[i].name, outfile, false, optional, use_db);
writeDir(outfile, files[i].name, files[i].created, files[i].last_modified_orig, files[i].file_permission_bits);
bool b=initialCheck(orig_dir+os_file_sep()+files[i].name, dir+os_file_sep()+files[i].name, named_path+os_file_sep()+files[i].name, outfile, false, optional, use_db);
outfile << "d\"..\"\n";
if(!b)
@ -993,6 +1015,9 @@ void IndexThread::readBackupDirs(void)
namespace
{
const int64 WINDOWS_TICK=10000000;
const int64 SEC_TO_UNIX_EPOCH=11644473600LL;
std::vector<SFileAndHash> convertToFileAndHash(const std::vector<SFile> files)
{
std::vector<SFileAndHash> ret;
@ -1003,6 +1028,16 @@ namespace
ret[i].last_modified=files[i].last_modified;
ret[i].name=files[i].name;
ret[i].size=files[i].size;
#ifdef _WIN32
ret[i].last_modified_orig=files[i].last_modified/WINDOWS_TICK-SEC_TO_UNIX_EPOCH;
ret[i].created=files[i].created/WINDOWS_TICK-SEC_TO_UNIX_EPOCH;
#else
ret[i].last_modified_orig=files[i].last_modified;
ret[i].created=files[i].created;
#endif
}
return ret;
}
@ -1098,7 +1133,7 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
#else
use_db=false;
#endif
std::vector<SFileAndHash> tmp;
std::vector<SFileAndHash> fs_files;
if(use_db==false || it_dir!=changed_dirs.end())
{
++index_c_fs;
@ -1106,7 +1141,8 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
std::wstring tpath=os_file_prefix(path);
bool has_error;
tmp=convertToFileAndHash(getFiles(tpath, &has_error));
fs_files=convertToFileAndHash(getFiles(tpath, &has_error));
readPermissions(tpath, fs_files);
if(has_error)
{
@ -1151,40 +1187,40 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
if(!changed_files.empty())
{
for(size_t i=0;i<tmp.size();++i)
for(size_t i=0;i<fs_files.size();++i)
{
if(!tmp[i].isdir)
if(!fs_files[i].isdir)
{
if( std::binary_search(changed_files.begin(), changed_files.end(), tmp[i].name) )
if( std::binary_search(changed_files.begin(), changed_files.end(), fs_files[i].name) )
{
VSSLog(L"Found changed file: " + tmp[i].name, LL_DEBUG);
VSSLog(L"Found changed file: " + fs_files[i].name, LL_DEBUG);
tmp[i].last_modified*=Server->getRandomNumber();
if(tmp[i].last_modified>0)
tmp[i].last_modified*=-1;
else if(tmp[i].last_modified==0)
tmp[i].last_modified=-1;
fs_files[i].last_modified*=Server->getRandomNumber();
if(fs_files[i].last_modified>0)
fs_files[i].last_modified*=-1;
else if(fs_files[i].last_modified==0)
fs_files[i].last_modified=-1;
}
else
{
std::vector<SFileAndHash>::const_iterator it_db_file=std::lower_bound(db_files.begin(), db_files.end(), tmp[i]);
std::vector<SFileAndHash>::const_iterator it_db_file=std::lower_bound(db_files.begin(), db_files.end(), fs_files[i]);
if( it_db_file!=db_files.end()
&& (*it_db_file).name==tmp[i].name
&& (*it_db_file).isdir==tmp[i].isdir
&& (*it_db_file).name==fs_files[i].name
&& (*it_db_file).isdir==fs_files[i].isdir
&& (*it_db_file).last_modified<0 )
{
VSSLog(L"File changed at last backup: "+ tmp[i].name, LL_DEBUG);
VSSLog(L"File changed at last backup: "+ fs_files[i].name, LL_DEBUG);
if( tmp[i].last_modified<last_filebackup_filetime)
if( fs_files[i].last_modified<last_filebackup_filetime)
{
tmp[i].last_modified=it_db_file->last_modified;
fs_files[i].last_modified=it_db_file->last_modified;
}
else
{
VSSLog("Modification time indicates the file may have another change", LL_DEBUG);
tmp[i].last_modified*=Server->getRandomNumber();
if(tmp[i].last_modified>0)
tmp[i].last_modified*=-1;
fs_files[i].last_modified*=Server->getRandomNumber();
if(fs_files[i].last_modified>0)
fs_files[i].last_modified*=-1;
}
}
}
@ -1197,15 +1233,15 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
if(calculate_filehashes_on_client)
{
addMissingHashes(has_files ? &db_files : NULL, &tmp, orig_path, path, named_path);
addMissingHashes(has_files ? &db_files : NULL, &fs_files, orig_path, path, named_path);
}
if( has_files)
{
if(tmp!=db_files)
if(fs_files!=db_files)
{
++index_c_db_update;
modifyFilesInt(path_lower, tmp);
modifyFilesInt(path_lower, fs_files);
}
}
else
@ -1214,31 +1250,31 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
if(calculate_filehashes_on_client)
{
#endif
cd->addFiles(path_lower, tmp);
cd->addFiles(path_lower, fs_files);
#ifndef _WIN32
}
#endif
}
return tmp;
return fs_files;
}
#ifdef _WIN32
else
{
if( cd->getFiles(path_lower, tmp) )
if( cd->getFiles(path_lower, fs_files) )
{
++index_c_db;
if(calculate_filehashes_on_client)
{
if(addMissingHashes(&tmp, NULL, orig_path, path, named_path))
if(addMissingHashes(&fs_files, NULL, orig_path, path, named_path))
{
++index_c_db_update;
modifyFilesInt(path_lower, tmp);
modifyFilesInt(path_lower, fs_files);
}
}
return tmp;
return fs_files;
}
else
{
@ -1247,7 +1283,8 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
std::wstring tpath=os_file_prefix(path);
bool has_error;
tmp=convertToFileAndHash(getFiles(tpath, &has_error));
fs_files=convertToFileAndHash(getFiles(tpath, &has_error));
readPermissions(tpath, fs_files);
if(has_error)
{
if(os_directory_exists(index_root_path))
@ -1263,11 +1300,11 @@ std::vector<SFileAndHash> IndexThread::getFilesProxy(const std::wstring &orig_pa
if(calculate_filehashes_on_client)
{
addMissingHashes(NULL, &tmp, orig_path, path, named_path);
addMissingHashes(NULL, &fs_files, orig_path, path, named_path);
}
cd->addFiles(path_lower, tmp);
return tmp;
cd->addFiles(path_lower, fs_files);
return fs_files;
}
}
#endif
@ -2970,3 +3007,84 @@ std::string IndexThread::escapeListName( const std::string& listname )
}
return ret;
}
void IndexThread::writeTokens()
{
std::auto_ptr<ISettingsReader> access_keys(
Server->createFileSettingsReader("access_keys.properties"));
std::string access_keys_data;
std::vector<std::wstring> keys
= access_keys->getKeys();
bool has_server_key=false;
std::string curr_key;
for(size_t i=0;i<keys.size();++i)
{
if(keys[i]==L"key."+widen(starttoken))
{
has_server_key=true;
curr_key=wnarrow(access_keys->getValue(keys[i], std::wstring()));
}
access_keys_data+=wnarrow(keys[i])+"="+
wnarrow(access_keys->getValue(keys[i], std::wstring()))+"\n";
}
if(!has_server_key)
{
curr_key = Server->secureRandomString(30);
access_keys_data+="key."+starttoken+"="+
curr_key+"\n";
write_file_only_admin(access_keys_data, "access_keys.properties");
}
write_tokens();
std::vector<ClientDAO::SToken> tokens = cd->getFileAccessTokens();
std::string ids;
for(size_t i=0;i<tokens.size();++i)
{
if(!ids.empty())
{
ids+=",";
}
ids+=nconvert(tokens[i].id);
}
std::string data="ids="+ids+"\n";
data+="access_key="+curr_key+"\n";
for(size_t i=0;i<tokens.size();++i)
{
data+=nconvert(tokens[i].id)+".username="+base64_encode_dash(Server->ConvertToUTF8(tokens[i].username))+"\n";
data+=nconvert(tokens[i].id)+".token="+Server->ConvertToUTF8(tokens[i].token)+"\n";
}
write_file_only_admin(data, "urbackup"+os_file_sepn()+"data"+os_file_sepn()+"tokens_"+starttoken+".properties");
}
void IndexThread::readPermissions(const std::wstring& dir, std::vector<SFileAndHash>& files )
{
for(size_t i=0;i<files.size();++i)
{
readPermissions(dir + os_file_sep() + files[i].name, files[i].file_permission_bits);
}
}
void IndexThread::readPermissions( const std::wstring& path, std::string& permissions )
{
permissions = get_file_tokens(path, cd, token_cache);
}
void IndexThread::writeDir(std::fstream& out, const std::wstring& name, int64 creat, int64 mod, const std::string& permissions )
{
out << "d\"" << escapeListName(Server->ConvertToUTF8(name)) << "\""
"rpb=" << base64_encode_dash(permissions) <<
"&mod=" << nconvert(mod) <<
"&creat=" << nconvert(creat) << "\n";
}

11
urbackupclient/client.h
View File

@ -6,6 +6,7 @@
#include "../urbackupcommon/os_functions.h"
#include "clientdao.h"
#include <map>
#include "win_tokens.h"
#ifdef _WIN32
#ifndef VSS_XP
@ -181,6 +182,14 @@ private:
std::string escapeListName(const std::string& listname);
void writeTokens();
void readPermissions(const std::wstring& path, std::string& permissions);
void readPermissions(const std::wstring& dir, std::vector<SFileAndHash>& files);
void writeDir(std::fstream& out, const std::wstring& name, int64 creat, int64 mod, const std::string& permissions);
std::string starttoken;
std::vector<SBackupDir> backup_dirs;
@ -237,6 +246,8 @@ private:
std::vector<std::pair<std::string, int> > vsslog;
_i64 last_filebackup_filetime;
TokenCache token_cache;
};
std::wstring add_trailing_slash(const std::wstring &strDirName);

109
urbackupclient/clientdao.cpp
View File

@ -103,12 +103,18 @@ void ClientDAO::destroyQueries(void)
void ClientDAO::prepareQueriesGen(void)
{
q_updateShadowCopyStarttime=NULL;
q_updateFileAccessToken=NULL;
q_getFileAccessTokens=NULL;
q_getFileAccessTokenId=NULL;
}
//@-SQLGenDestruction
void ClientDAO::destroyQueriesGen(void)
{
db->destroyQuery(q_updateShadowCopyStarttime);
db->destroyQuery(q_updateFileAccessToken);
db->destroyQuery(q_getFileAccessTokens);
db->destroyQuery(q_getFileAccessTokenId);
}
void ClientDAO::restartQueries(void)
@ -166,6 +172,24 @@ bool ClientDAO::getFiles(std::wstring path, std::vector<SFileAndHash> &data)
ptr+=hashsize;
unsigned short fpb_size;
memcpy(&fpb_size, ptr, sizeof(unsigned short));
ptr+=sizeof(unsigned short);
f.file_permission_bits.resize(fpb_size);
if(fpb_size>0)
{
memcpy(&f.file_permission_bits[0], ptr, fpb_size);
}
ptr+=fpb_size;
memcpy(&f.last_modified_orig, ptr, sizeof(int64));
ptr+=sizeof(int64);
memcpy(&f.created, ptr, sizeof(int64));
ptr+=sizeof(int64);
data.push_back(f);
}
return true;
@ -185,6 +209,9 @@ char * constructData(const std::vector<SFileAndHash> &data, size_t &datasize)
++datasize;
datasize+=sizeof(unsigned short);
datasize+=data[i].hash.size();
datasize+=sizeof(unsigned short);
datasize+=data[i].file_permission_bits.size();
datasize+=sizeof(int64)*2;
}
char *buffer=new char[datasize];
char *ptr=buffer;
@ -209,6 +236,15 @@ char * constructData(const std::vector<SFileAndHash> &data, size_t &datasize)
ptr+=sizeof(hashsize);
memcpy(ptr, data[i].hash.data(), hashsize);
ptr+=hashsize;
unsigned short fpb_size=static_cast<unsigned short>(data[i].file_permission_bits.size());
memcpy(ptr, &fpb_size, sizeof(fpb_size));
ptr+=sizeof(fpb_size);
memcpy(ptr, data[i].file_permission_bits.data(), fpb_size);
ptr+=fpb_size;
memcpy(ptr, (char*)&data[i].last_modified_orig, sizeof(int64));
ptr+=sizeof(int64);
memcpy(ptr, (char*)&data[i].created, sizeof(int64));
ptr+=sizeof(int64);
}
return buffer;
}
@ -570,3 +606,76 @@ void ClientDAO::updateShadowCopyStarttime(int id)
q_updateShadowCopyStarttime->Reset();
}
/**
* @-SQLGenAccess
* @func void ClientDAO::updateFileAccessToken
* @sql
* INSERT OR REPLACE INTO fileaccess_tokens
* (username, token)
* VALUES
* (:username(string), :token(string))
*/
void ClientDAO::updateFileAccessToken(const std::wstring& username, const std::wstring& token)
{
if(q_updateFileAccessToken==NULL)
{
q_updateFileAccessToken=db->Prepare("INSERT OR REPLACE INTO fileaccess_tokens (username, token) VALUES (?, ?)", false);
}
q_updateFileAccessToken->Bind(username);
q_updateFileAccessToken->Bind(token);
q_updateFileAccessToken->Write();
q_updateFileAccessToken->Reset();
}
/**
* @-SQLGenAccess
* @func vector<SToken> ClientDAO::getFileAccessTokens
* @return int64 id, string username, string token
* @sql
* SELECT id, username, token
* FROM fileaccess_tokens
*/
std::vector<ClientDAO::SToken> ClientDAO::getFileAccessTokens(void)
{
if(q_getFileAccessTokens==NULL)
{
q_getFileAccessTokens=db->Prepare("SELECT id, username, token FROM fileaccess_tokens", false);
}
db_results res=q_getFileAccessTokens->Read();
std::vector<ClientDAO::SToken> ret;
ret.resize(res.size());
for(size_t i=0;i<res.size();++i)
{
ret[i].id=watoi64(res[i][L"id"]);
ret[i].username=res[i][L"username"];
ret[i].token=res[i][L"token"];
}
return ret;
}
/**
* @-SQLGenAccess
* @func int64 ClientDAO::getFileAccessTokenId
* @return int64 id
* @sql
* SELECT id
* FROM fileaccess_tokens WHERE username = :username(string)
*/
ClientDAO::CondInt64 ClientDAO::getFileAccessTokenId(const std::wstring& username)
{
if(q_getFileAccessTokenId==NULL)
{
q_getFileAccessTokenId=db->Prepare("SELECT id FROM fileaccess_tokens WHERE username = ?", false);
}
q_getFileAccessTokenId->Bind(username);
db_results res=q_getFileAccessTokenId->Read();
q_getFileAccessTokenId->Reset();
CondInt64 ret = { false, 0 };
if(!res.empty())
{
ret.exists=true;
ret.value=watoi64(res[0][L"id"]);
}
return ret;
}

27
urbackupclient/clientdao.h
View File

@ -1,3 +1,5 @@
#pragma once
#include "../Interface/Database.h"
#include "../Interface/Query.h"
#include "../urbackupcommon/os_functions.h"
@ -68,6 +70,9 @@ struct SFileAndHash
int64 last_modified;
bool isdir;
std::string hash;
std::string file_permission_bits;
int64 last_modified_orig;
int64 created;
bool operator<(const SFileAndHash &other) const
{
@ -80,7 +85,10 @@ struct SFileAndHash
size == other.size &&
last_modified == other.last_modified &&
isdir == other.isdir &&
hash == other.hash;
hash == other.hash &&
file_permission_bits == other.file_permission_bits &&
last_modified_orig == other.last_modified_orig &&
created == other.created;
}
};
@ -143,9 +151,23 @@ public:
void updateMiscValue(const std::string& key, const std::wstring& value);
//@-SQLGenFunctionsBegin
struct CondInt64
{
bool exists;
int64 value;
};
struct SToken
{
int64 id;
std::wstring username;
std::wstring token;
};
void updateShadowCopyStarttime(int id);
void updateFileAccessToken(const std::wstring& username, const std::wstring& token);
std::vector<SToken> getFileAccessTokens(void);
CondInt64 getFileAccessTokenId(const std::wstring& username);
//@-SQLGenFunctionsEnd
private:
@ -187,5 +209,8 @@ private:
//@-SQLGenVariablesBegin
IQuery* q_updateShadowCopyStarttime;
IQuery* q_updateFileAccessToken;
IQuery* q_getFileAccessTokens;
IQuery* q_getFileAccessTokenId;
//@-SQLGenVariablesEnd
};

20
urbackupclient/dllmain.cpp
View File

@ -301,7 +301,7 @@ DLLEXPORT void LoadActions(IServer* pServer)
internetclient_ticket=InternetClient::start(do_leak_check);
Server->Log("Started UrBackupClient Backend...", LL_INFO);
Server->wait(1000);
Server->wait(1000);
}
DLLEXPORT void UnloadActions(void)
@ -410,6 +410,16 @@ void update_client12_13(IDatabase *db)
db->Write("UPDATE backupdirs SET optional=0 WHERE optional IS NULL");
}
void update_client13_14(IDatabase *db)
{
db->Write("CREATE TABLE fileaccess_tokens (id INTEGER PRIMARY KEY, username TEXT UNIQUE, token TEXT)");
}
void update_client14_15(IDatabase *db)
{
db->Write("DELETE FROM files");
}
bool upgrade_client(void)
{
IDatabase *db=Server->getDatabase(Server->getThreadID(), URBACKUPDB_CLIENT);
@ -479,6 +489,14 @@ bool upgrade_client(void)
update_client12_13(db);
++ver;
break;
case 13:
update_client13_14(db);
++ver;
break;
case 14:
update_client14_15(db);
++ver;
break;
default:
break;
}

2
urbackupclient/urbackupclient.vcxproj
View File

@ -82,6 +82,7 @@
<ClCompile Include="watchdir\DirectoryChanges.cpp" />
<ClCompile Include="win_all_volumes.cpp" />
<ClCompile Include="win_sysvol.cpp" />
<ClCompile Include="win_tokens.cpp" />
<ClCompile Include="win_ver.cpp" />
</ItemGroup>
<ItemGroup>
@ -114,6 +115,7 @@
<ClInclude Include="watchdir\Event.h" />
<ClInclude Include="win_all_volumes.h" />
<ClInclude Include="win_sysvol.h" />
<ClInclude Include="win_tokens.h" />
<ClInclude Include="win_ver.h" />
</ItemGroup>
<PropertyGroup Label="Globals">

6
urbackupclient/urbackupclient.vcxproj.filters
View File

@ -120,6 +120,9 @@
<ClCompile Include="win_all_volumes.cpp">
<Filter>Quelldateien</Filter>
</ClCompile>
<ClCompile Include="win_tokens.cpp">
<Filter>Quelldateien</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="watchdir\CriticalSection.h">
@ -212,5 +215,8 @@
<ClInclude Include="win_all_volumes.h">
<Filter>Headerdateien</Filter>
</ClInclude>
<ClInclude Include="win_tokens.h">
<Filter>Headerdateien</Filter>
</ClInclude>
</ItemGroup>
</Project>

360
urbackupclient/win_tokens.cpp Normal file
View File

@ -0,0 +1,360 @@
#include <Windows.h>
#include <LM.h>
#include <Sddl.h>
#include "../Interface/Server.h"
#include "../stringtools.h"
#include "../urbackupcommon/os_functions.h"
#include "database.h"
#include "clientdao.h"
#include "Aclapi.h"
#include "win_tokens.h"
#pragma comment(lib, "netapi32.lib")
struct Token
{
int64 id;
std::vector<char> account_buf;
};
struct TokenCacheInt
{
std::vector<Token> tokens;
int64 max_id;
};
TokenCache::TokenCache( const TokenCache& other )
{
}
TokenCache::TokenCache() : token_cache(NULL)
{
}
TokenCache::~TokenCache()
{
}
void TokenCache::operator=(const TokenCache& other )
{
}
TokenCacheInt* TokenCache::get()
{
return token_cache.get();
}
void TokenCache::reset(TokenCacheInt* cache)
{
token_cache.reset(cache);
}
std::vector<std::wstring> get_users()
{
LPUSER_INFO_0 buf;
DWORD prefmaxlen = MAX_PREFERRED_LENGTH;
DWORD entriesread = 0;
DWORD totalentries = 0;
DWORD resume_handle = 0;
NET_API_STATUS status;
std::vector<std::wstring> ret;
do
{
status = NetUserEnum(NULL, 0, FILTER_NORMAL_ACCOUNT,
(LPBYTE*)&buf, prefmaxlen, &entriesread,
&totalentries, &resume_handle);
if (status == NERR_Success || status == ERROR_MORE_DATA)
{
for(DWORD i=0;i<entriesread;++i)
{
USER_INFO_0 user_info = *(buf+i);
ret.push_back(user_info.usri0_name);
}
}
else
{
Server->Log("Error while enumerating users: "+ nconvert((int)status), LL_ERROR);
}
if(buf!=NULL)
{
NetApiBufferFree(buf);
}
}
while (status == ERROR_MORE_DATA);
return ret;
}
bool write_tokens()
{
IDatabase* db = Server->getDatabase(Server->getThreadID(), URBACKUPDB_CLIENT);
ClientDAO dao(db);
char hostname_c[MAX_PATH];
hostname_c[0]=0;
gethostname(hostname_c, MAX_PATH);
std::wstring hostname = widen(hostname_c);
if(!hostname.empty())
hostname+=L"\\";
db->BeginTransaction();
bool has_new_token=false;
std::vector<std::wstring> users = get_users();
os_create_dir("tokens");
std::vector<SFile> files = getFiles(L"tokens", NULL, false, false);
for(size_t i=0;i<users.size();++i)
{
std::wstring token_fn = L"tokens" + os_file_sep()+users[i];
bool file_found=false;
for(size_t j=0;j<files.size();++j)
{
if(files[j].name==users[i])
{
file_found=true;
break;
}
}
if(file_found)
{
continue;
}
std::vector<char> sid_buffer;
sid_buffer.resize(sizeof(SID));
DWORD account_sid_size = sizeof(SID);
SID_NAME_USE sid_name_use;
std::wstring referenced_domain;
referenced_domain.resize(1);
DWORD referenced_domain_size = 1;
BOOL b=LookupAccountNameW(NULL, (hostname+users[i]).c_str(),
&sid_buffer[0], &account_sid_size, &referenced_domain[0],
&referenced_domain_size, &sid_name_use);
if(!b && GetLastError()==ERROR_INSUFFICIENT_BUFFER)
{
referenced_domain.resize(referenced_domain_size);
sid_buffer.resize(account_sid_size);
b=LookupAccountNameW(NULL, (hostname+users[i]).c_str(),
&sid_buffer[0], &account_sid_size, &referenced_domain[0],
&referenced_domain_size, &sid_name_use);
}
if(!b)
{
Server->Log("Error getting accout SID. Errorcode: "+nconvert((int)GetLastError()), LL_ERROR);
continue;
}
SID* account_sid = reinterpret_cast<SID*>(&sid_buffer[0]);
LPWSTR str_account_sid;
b = ConvertSidToStringSidW(account_sid, &str_account_sid);
if(!b)
{
Server->Log("Error converting SID to string SID. Errorcode: "+nconvert((int)GetLastError()), LL_ERROR);
continue;
}
std::wstring dacl = std::wstring(L"D:(A;OICI;GA;;;") + str_account_sid + L")";
LocalFree(str_account_sid);
SECURITY_ATTRIBUTES sa;
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.bInheritHandle = FALSE;
b=ConvertStringSecurityDescriptorToSecurityDescriptor(
dacl.c_str(),
SDDL_REVISION_1,
&(sa.lpSecurityDescriptor),
NULL);
if(!b)
{
Server->Log("Error creating security descriptor. Errorcode: "+nconvert((int)GetLastError()), LL_ERROR);
continue;
}
HANDLE file = CreateFileW(token_fn.c_str(),
GENERIC_READ | GENERIC_WRITE, 0, &sa, CREATE_ALWAYS, 0, NULL);
if(file==INVALID_HANDLE_VALUE)
{
Server->Log("Error opening file. Errorcode: "+nconvert((int)GetLastError()), LL_ERROR);
LocalFree(sa.lpSecurityDescriptor);
continue;
}
std::string token = Server->secureRandomString(20);
dao.updateFileAccessToken(users[i], widen(token));
has_new_token=true;
DWORD written=0;
while(written<token.size())
{
b = WriteFile(file, token.data()+written, static_cast<DWORD>(token.size())-written, &written, NULL);
if(!b)
{
Server->Log("Error writing to token file. Errorcode: "+nconvert((int)GetLastError()), LL_ERROR);
CloseHandle(file);
LocalFree(sa.lpSecurityDescriptor);
continue;
}
}
CloseHandle(file);
LocalFree(sa.lpSecurityDescriptor);
}
if(has_new_token)
{
dao.updateMiscValue("has_new_token", L"true");
}
db->EndTransaction();
return has_new_token;
}
void read_all_tokens(ClientDAO* dao, TokenCache& token_cache)
{
TokenCacheInt* cache = new TokenCacheInt;
token_cache.reset(cache);
cache->max_id=-1;
std::vector<std::wstring> users = get_users();
char hostname_c[MAX_PATH];
hostname_c[0]=0;
gethostname(hostname_c, MAX_PATH);
std::wstring hostname = widen(hostname_c);
if(!hostname.empty())
hostname+=L"\\";
for(size_t i=0;i<users.size();++i)
{
Token new_token;
new_token.account_buf.resize(sizeof(SID));
DWORD account_sid_size = sizeof(SID);
SID_NAME_USE sid_name_use;
std::wstring referenced_domain;
referenced_domain.resize(1);
DWORD referenced_domain_size = 1;
BOOL b=LookupAccountNameW(NULL, (hostname+users[i]).c_str(),
&new_token.account_buf[0], &account_sid_size, &referenced_domain[0],
&referenced_domain_size, &sid_name_use);
if(!b && GetLastError()==ERROR_INSUFFICIENT_BUFFER)
{
referenced_domain.resize(referenced_domain_size);
new_token.account_buf.resize(account_sid_size);
b=LookupAccountNameW(NULL, (hostname+users[i]).c_str(),
&new_token.account_buf[0], &account_sid_size, &referenced_domain[0],
&referenced_domain_size, &sid_name_use);
}
ClientDAO::CondInt64 token_id = dao->getFileAccessTokenId(users[i]);
if(!token_id.exists)
{
Server->Log("Token id for user not found", LL_ERROR);
continue;
}
new_token.id=token_id.value;
cache->tokens.push_back(new_token);
if(new_token.id>cache->max_id)
{
cache->max_id=new_token.id;
}
}
}
void set_bit(int64 bit_num, std::string& bits)
{
size_t bitmap_byte=(size_t)(bit_num/8);
size_t bitmap_bit=bit_num%8;
unsigned char b=static_cast<unsigned char>(bits[bitmap_byte]);
b=b|(1<<(7-bitmap_bit));
bits[bitmap_byte]=static_cast<char>(b);
}
std::string get_file_tokens( const std::wstring& fn, ClientDAO* dao, TokenCache& token_cache )
{
if(token_cache.get()==NULL)
{
read_all_tokens(dao, token_cache);
}
TokenCacheInt* cache = token_cache.get();
PACL dacl;
DWORD rc = GetNamedSecurityInfoW(fn.c_str(), SE_FILE_OBJECT,
DACL_SECURITY_INFORMATION, NULL, NULL, &dacl,
NULL, NULL);
if(rc!=ERROR_SUCCESS)
{
Server->Log(L"Error getting DACL of file \""+fn+L"\". Errorcode: "+convert((int)rc), LL_ERROR);
return std::string();
}
std::string bits;
bits.resize(cache->max_id/8 + (cache->max_id%8?1:0));
for(size_t i=0;i<cache->tokens.size();++i)
{
Token& token = cache->tokens[i];
TRUSTEE trustee;
BuildTrusteeWithSid(&trustee, reinterpret_cast<SID*>(&token.account_buf[0]));
ACCESS_MASK access_mask;
rc = GetEffectiveRightsFromAclW(dacl, &trustee, &access_mask);
if(rc!=ERROR_SUCCESS)
{
Server->Log(L"Error getting effective rights. Errorcode: "+convert((int)rc), LL_ERROR);
continue;
}
if(access_mask & GENERIC_READ || access_mask & GENERIC_ALL
|| access_mask & FILE_GENERIC_READ || access_mask & FILE_ALL_ACCESS)
{
set_bit(token.id, bits);
}
}
return bits;
}
void free_tokencache( TokenCacheInt* cache )
{
delete cache;
}

28
urbackupclient/win_tokens.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include "clientdao.h"
#include <memory>
bool write_tokens();
struct TokenCacheInt;
class TokenCache
{
public:
TokenCache();
~TokenCache();
void reset(TokenCacheInt* cache=NULL);
TokenCacheInt* get();
private:
void operator=(const TokenCache& other);
TokenCache(const TokenCache& other);
std::auto_ptr<TokenCacheInt> token_cache;
};
std::string get_file_tokens(const std::wstring& fn, ClientDAO* dao, TokenCache& cache);

3
urbackupcommon/capa_bits.h
View File

@ -10,4 +10,5 @@ const int DONT_ALLOW_STARTING_INCR_IMAGE_BACKUPS=1<<7;
const int DONT_ALLOW_STARTING_FULL_IMAGE_BACKUPS=1<<8;
const int DONT_ALLOW_STARTING_INCR_FILE_BACKUPS=1<<9;
const int DONT_ALLOW_STARTING_FULL_FILE_BACKUPS=1<<10;
const int DONT_ALLOW_EXIT_TRAY_ICON=1<<11;
const int DONT_ALLOW_EXIT_TRAY_ICON=1<<11;
const int ALLOW_TOKEN_AUTHENTICATION=1<<12;

3
urbackupcommon/os_functions.h
View File

@ -13,6 +13,7 @@ struct SFile
std::wstring name;
int64 size;
int64 last_modified;
int64 created;
bool isdir;
bool operator<(const SFile &other) const
@ -23,6 +24,8 @@ struct SFile
std::vector<SFile> getFiles(const std::wstring &path, bool *has_error=NULL, bool follow_symlinks=false, bool exact_filesize=true);
SFile getFileMetadata(const std::wstring &path);
void removeFile(const std::wstring &path);
void moveFile(const std::wstring &src, const std::wstring &dst);

49
urbackupcommon/os_functions_win.cpp
View File

@ -140,6 +140,10 @@ std::vector<SFile> getFiles(const std::wstring &path, bool *has_error, bool foll
size.LowPart=wfd.nFileSizeLow;
f.size=size.QuadPart;
lwt.HighPart=wfd.ftCreationTime.dwHighDateTime;
lwt.LowPart=wfd.ftCreationTime.dwLowDateTime;
f.created=lwt.QuadPart;
if(exact_filesize && !(wfd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) )
{
if(wfd.dwFileAttributes &FILE_ATTRIBUTE_REPARSE_POINT)
@ -160,6 +164,11 @@ std::vector<SFile> getFiles(const std::wstring &path, bool *has_error, bool foll
lwt.LowPart = file_info.ftLastWriteTime.dwLowDateTime;
f.last_modified = lwt.QuadPart;
lwt.HighPart=file_info.ftCreationTime.dwHighDateTime;
lwt.LowPart=file_info.ftCreationTime.dwLowDateTime;
f.created=lwt.QuadPart;
}
CloseHandle(hFile);
@ -178,6 +187,11 @@ std::vector<SFile> getFiles(const std::wstring &path, bool *has_error, bool foll
lwt.LowPart = fad.ftLastWriteTime.dwLowDateTime;
f.last_modified = lwt.QuadPart;
lwt.HighPart=fad.ftCreationTime.dwHighDateTime;
lwt.LowPart=fad.ftCreationTime.dwLowDateTime;
f.created=lwt.QuadPart;
}
}
}
@ -194,6 +208,41 @@ std::vector<SFile> getFiles(const std::wstring &path, bool *has_error, bool foll
return tmp;
}
SFile getFileMetadata( const std::wstring &path )
{
SFile ret;
ret.name=path;
WIN32_FILE_ATTRIBUTE_DATA fad;
if( GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &fad) )
{
if (fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
{
ret.isdir=true;
}
LARGE_INTEGER size;
size.HighPart = fad.nFileSizeHigh;
size.LowPart = fad.nFileSizeLow;
ret.size = size.QuadPart;
LARGE_INTEGER lwt;
lwt.HighPart = fad.ftLastWriteTime.dwHighDateTime;
lwt.LowPart = fad.ftLastWriteTime.dwLowDateTime;
ret.last_modified = lwt.QuadPart;
lwt.HighPart=fad.ftCreationTime.dwHighDateTime;
lwt.LowPart=fad.ftCreationTime.dwLowDateTime;
ret.created=lwt.QuadPart;
return ret;
}
else
{
return SFile();
}
}
#ifndef OS_FUNC_NO_SERVER
void removeFile(const std::wstring &path)
{

3
urbackupcommon/settingslist.cpp
View File

@ -60,6 +60,7 @@ std::vector<std::wstring> getSettingsList(void)
ret.push_back(L"internet_calculate_filehashes_on_client");
ret.push_back(L"compress_images");
ret.push_back(L"internet_connect_always");
ret.push_back(L"server_url");
return ret;
}
@ -88,6 +89,7 @@ std::vector<std::wstring> getGlobalizedSettingsList(void)
std::vector<std::wstring> ret;
ret.push_back(L"internet_server");
ret.push_back(L"internet_server_port");
ret.push_back(L"server_url");
return ret;
}
@ -124,5 +126,6 @@ std::vector<std::wstring> getGlobalSettingsList(void)
ret.push_back(L"filescache_size");
ret.push_back(L"trust_client_hashes");
ret.push_back(L"show_server_updates");
ret.push_back(L"server_url");
return ret;
}

82
urbackupserver/dao/ServerBackupDao.cpp
View File

@ -661,6 +661,70 @@ std::vector<ServerBackupDao::SDuration> ServerBackupDao::getLastFullDurations(in
return ret;
}
/**
* @-SQLGenAccess
* @func string ServerBackupDao::getSetting
* @return string value
* @sql
* SELECT value FROM settings_db.settings WHERE clientid=:clientid(int) AND key=:key(string)
*/
ServerBackupDao::CondString ServerBackupDao::getSetting(int clientid, const std::wstring& key)
{
if(q_getSetting==NULL)
{
q_getSetting=db->Prepare("SELECT value FROM settings_db.settings WHERE clientid=? AND key=?", false);
}
q_getSetting->Bind(clientid);
q_getSetting->Bind(key);
db_results res=q_getSetting->Read();
q_getSetting->Reset();
CondString ret = { false, L"" };
if(!res.empty())
{
ret.exists=true;
ret.value=res[0][L"value"];
}
return ret;
}
/**
* @-SQLGenAccess
* @func void ServerBackupDao::insertSetting
* @sql
* INSERT INTO settings_db.settings (key, value, clientid) VALUES ( :key(string), :value(string), :clientid(int) )
*/
void ServerBackupDao::insertSetting(const std::wstring& key, const std::wstring& value, int clientid)
{
if(q_insertSetting==NULL)
{
q_insertSetting=db->Prepare("INSERT INTO settings_db.settings (key, value, clientid) VALUES ( ?, ?, ? )", false);
}
q_insertSetting->Bind(key);
q_insertSetting->Bind(value);
q_insertSetting->Bind(clientid);
q_insertSetting->Write();
q_insertSetting->Reset();
}
/**
* @-SQLGenAccess
* @func void ServerBackupDao::updateSetting
* @sql
* UPDATE settings_db.settings SET value=:value(string) WHERE key=:key(string) AND clientid=:clientid(int)
*/
void ServerBackupDao::updateSetting(const std::wstring& value, const std::wstring& key, int clientid)
{
if(q_updateSetting==NULL)
{
q_updateSetting=db->Prepare("UPDATE settings_db.settings SET value=? WHERE key=? AND clientid=?", false);
}
q_updateSetting->Bind(value);
q_updateSetting->Bind(key);
q_updateSetting->Bind(clientid);
q_updateSetting->Write();
q_updateSetting->Reset();
}
//@-SQLGenSetup
void ServerBackupDao::prepareQueries( void )
{
@ -693,6 +757,9 @@ void ServerBackupDao::prepareQueries( void )
q_getOrigClientSettings=NULL;
q_getLastIncrementalDurations=NULL;
q_getLastFullDurations=NULL;
q_getSetting=NULL;
q_insertSetting=NULL;
q_updateSetting=NULL;
}
//@-SQLGenDestruction
@ -727,6 +794,9 @@ void ServerBackupDao::destroyQueries( void )
db->destroyQuery(q_getOrigClientSettings);
db->destroyQuery(q_getLastIncrementalDurations);
db->destroyQuery(q_getLastFullDurations);
db->destroyQuery(q_getSetting);
db->destroyQuery(q_insertSetting);
db->destroyQuery(q_updateSetting);
}
void ServerBackupDao::commit()
@ -738,3 +808,15 @@ int64 ServerBackupDao::getLastId()
{
return db->getLastInsertID();
}
void ServerBackupDao::updateOrInsertSetting( int clientid, const std::wstring& key, const std::wstring& value )
{
if(getSetting(clientid, key).exists)
{
updateSetting(value, key, clientid);
}
else
{
insertSetting(key, value, clientid);
}
}

8
urbackupserver/dao/ServerBackupDao.h
View File

@ -71,8 +71,13 @@ public:
CondString getOrigClientSettings(int clientid);
std::vector<SDuration> getLastIncrementalDurations(int clientid);
std::vector<SDuration> getLastFullDurations(int clientid);
CondString getSetting(int clientid, const std::wstring& key);
void insertSetting(const std::wstring& key, const std::wstring& value, int clientid);
void updateSetting(const std::wstring& value, const std::wstring& key, int clientid);
//@-SQLGenFunctionsEnd
void updateOrInsertSetting(int clientid, const std::wstring& key, const std::wstring& value);
private:
ServerBackupDao(ServerBackupDao& other) {}
void operator=(ServerBackupDao& other) {}
@ -110,6 +115,9 @@ private:
IQuery* q_getOrigClientSettings;
IQuery* q_getLastIncrementalDurations;
IQuery* q_getLastFullDurations;
IQuery* q_getSetting;
IQuery* q_insertSetting;
IQuery* q_updateSetting;
//@-SQLGenVariablesEnd
IDatabase *db;

17
urbackupserver/doc/admin_guide.tex
View File

@ -18,6 +18,8 @@
\tableofcontents
\newpage
\section{Introduction}
UrBackup is a client/server backup system. This means there exists a server
@ -286,6 +288,8 @@ Volume\textgreater being the drive letter of the backuped partition and YY the
current year, MM the current month, DD the current day in the month and HHMM the
hour and minute the image backup was started.
Since UrBackup Server 1.4 the VHD-files are compressed by default. This can be disabled in the image backup settings section. On Windows there are no tools to directly mount compressed VHD files. For mounting them on Linux see section \ref{sec:mounting_image_files}. For decompressing the image files such that they can be mounted on Windows see \ref{sec:decompressing_vhd_files}. The compressed VHD files have the extension '.vhdz'. The VHD files are compressed in 2MB blocks using GZIP compression with normal compression level.
\subsection{Collission propabilities}
In this section we will look at the probability that the UrBackup backup system considers data the same, even though it is different. This can be caused by a hash collision (data has the same hash, even though the data is different). If happening, a collision can lead to files being incorrectly linked or blocks in image backups not transferred.
@ -294,7 +298,7 @@ In this section we will look at the probability that the UrBackup backup system
UrBackup uses SHA512 to hash the files before file deduplication. In comparison ZFS uses SHA256 for block deduplication. The choice of SHA512 is definitely on the save side. The Wikipedia page for ``Birthday attack'' has a propability table for SHA512. According to it one needs $1.6*10^{68}$ different files (of same size) to reach a probability of $10^{-18}$ of a collision. It also states that $10^{-18}$ is the best case uncorrectable bit error rate of a typical hard disk. To have $1.6*10^{68}$ different files of $1KB$ you need $1.4551915*10^{56}$ EB of hard disk space. So it is ridiculously more likely that the hard disk returns bad data or the data gets corrupted in RAM, rather than UrBackup linking the wrong files to each other.
\subsection{Image backup collision probability}
\subsubsection{Image backup collision probability}
For the blocks in an image backup SHA256 is used. They are 256kbyte in size. The chance of a hash collision with SHA256 is $1$:($2^{256}$) ($p=\frac{1}{2^{256}}$) for two hashes. In the worst case you have $2TB/256kbyte = 8388608$ different blocks in an incremental image backup. The chance of having a collision in any of the 8388608 blocks (the worst case) is then $1-(1-\frac{1}{2^{256}})^{8388608} \approx 7*10^{-71}$. This is again ridiculously low compared to e.g. the probability of $10^{-18}$ of a typical hard disk having a uncorrectable bit error.
@ -565,6 +569,8 @@ Allow client-side pausing of backups & Allow the client(s) to pause backups & Ch
\hline
Allow client-side changing of settings & If this option is checked the clients can change their client specific settings via the client interface. If you do not check this the server settings always override the clients' settings. & Checked\\
\hline
Allow clients to quit the tray icon & Allow the client(s) to quit the tray icon. If the tray icon is quit current and future backups are paused. & Checked \\
\hline
\end{longtable}
\subsubsection{Backup window}
@ -614,6 +620,11 @@ gives the "Users" directory the name "User files" and the "Program files" direct
Those locations are only the default locations. Even if you check "Separate settings for this client" and disable "Allow client to change settings", once the client modified the paths, changes in this field are not used by the client any more.
Since UrBackup Client 1.4, if a directory to backup is not available the whole backup process is interuppted and the backup fails. You can mark directories such that the client continues if they are not present. With the above example you can do that by appending \textsl{/optional} to the location name as follows:
\begin{verbatim}
C:\Users|User files/optional;C:\Program Files|Programs/optional
\end{verbatim}
\subsection{Internet settings}
\begin{tabular}{|p{0.2\textwidth}|p{0.6\textwidth}|p{0.1\textwidth}|}
@ -639,6 +650,8 @@ Compressed transfer & If checked all data between server and clients is compress
Calculate file-hashes on the client & If checked the client calculates hashes for each file before the backups (only hashes of changed files are calculated).
The file then does not have to be transfered if another client already transfered the same file & Not checked \\
\hline
Connect to Internet backup server if connected to local backup server & If checked the client will connect to the configured Internet server, even if it is connect to a backup server on the local network. & Not checked \\
\hline
\end{tabular}
\subsection{Advanced settings}
@ -859,6 +872,7 @@ Port & Usage \\
\end{tabular}
\subsection{Mounting (compressed) VHD files on GNU/Linux}
\label{sec:mounting_image_files}
If you compiled UrBackup with fuse support or installed the debian packages the UrBackup server can mount
VHD(Z) files directly. You compile UrBackup with fuse support by configuring:
@ -874,6 +888,7 @@ All files in the backed up ``C'' volume will then be available read-only in \tex
Unmount the mounts created by UrBackup (see output of \textsl{mount}), to stop the background process.
\subsection{Decompress VHD files}
\label{sec:decompressing_vhd_files}
If you want to mount the VHD files on Windows and they are compressed (file extension is VHDZ), you need
to decompress them first. Use \textsl{C:\textbackslash Program Files\textbackslash UrBackupServer\textbackslash uncompress\_image.bat} for that.

288
urbackupserver/file_metadata.cpp Normal file
View File

@ -0,0 +1,288 @@
#include "file_metadata.h"
#include "../Interface/Server.h"
#include "../Interface/File.h"
#include "../urbackupcommon/os_functions.h"
#include "../stringtools.h"
#include "server_prepare_hash.h"
#include "../common/data.h"
#include "../fileservplugin/chunk_settings.h"
#include <assert.h>
#include <algorithm>
namespace
{
int64 get_hashdata_size(int64 hashfilesize)
{
int64 num_chunks = hashfilesize/c_checkpoint_dist+((hashfilesize%c_checkpoint_dist!=0)?1:0);
return chunkhash_file_off+num_chunks*chunkhash_single_size;
}
bool write_metadata(IFile* out, INotEnoughSpaceCallback *cb, const FileMetadata& metadata)
{
CWData data;
data.addChar(0);
metadata.serialize(data);
_u32 metadata_size = static_cast<_u32>(data.getDataSize());
metadata_size = little_endian(metadata_size);
if(!BackupServerPrepareHash::writeRepeatFreeSpace(out, reinterpret_cast<char*>(&metadata_size), sizeof(metadata_size), cb))
{
Server->Log(L"Error writing file metadata to file \""+out->getFilenameW()+L"\" -1", LL_ERROR);
return false;
}
if(!BackupServerPrepareHash::writeRepeatFreeSpace(out, data.getDataPtr(), data.getDataSize(), cb))
{
Server->Log(L"Error writing file metadata to file \""+out->getFilenameW()+L"\"", LL_ERROR);
return false;
}
return true;
}
bool read_metadata_values(IFile* in, FileMetadata& metadata)
{
_u32 metadata_size;
if(in->Read(reinterpret_cast<char*>(&metadata_size), sizeof(metadata_size))!=sizeof(metadata_size) ||
metadata_size==0)
{
Server->Log(L"Error reading file metadata hashfilesize from \""+in->getFilenameW()+L"\" -2", LL_DEBUG);
return false;
}
metadata_size = little_endian(metadata_size);
std::vector<char> buffer;
buffer.resize(metadata_size);
if(in->Read(&buffer[0], metadata_size)!=metadata_size)
{
Server->Log(L"Error reading file metadata hashfilesize from \""+in->getFilenameW()+L"\" -3", LL_ERROR);
return false;
}
CRData data(&buffer[0], buffer.size());
bool ok=true;
char version = 0;
ok &= data.getChar(&version);
ok &= metadata.read(data);
if(version!=0)
{
Server->Log(L"Unknown metadata version at \""+in->getFilenameW()+L"\"", LL_ERROR);
return false;
}
if(!ok)
{
Server->Log(L"Malformed metadata at \""+in->getFilenameW()+L"\"", LL_ERROR);
return false;
}
return true;
}
bool read_metadata(IFile* in, FileMetadata& metadata)
{
int64 hashfilesize;
in->Seek(0);
if(in->Read(reinterpret_cast<char*>(&hashfilesize), sizeof(hashfilesize))!=sizeof(hashfilesize))
{
Server->Log(L"Error reading file metadata hashfilesize from \""+in->getFilenameW()+L"\"", LL_DEBUG);
return false;
}
hashfilesize=little_endian(hashfilesize);
if(hashfilesize!=-1)
{
in->Seek(get_hashdata_size(hashfilesize));
}
return read_metadata_values(in, metadata);
}
}
bool write_file_metadata(const std::wstring& out_fn, INotEnoughSpaceCallback *cb, const FileMetadata& metadata)
{
std::auto_ptr<IFile> out(Server->openFile(os_file_prefix(out_fn), MODE_RW_CREATE));
if(!out.get())
{
Server->Log(L"Error writing file metadata to file \""+out_fn+L"\"", LL_ERROR);
return false;
}
int64 hashfilesize;
out->Seek(0);
if(out->Read(reinterpret_cast<char*>(&hashfilesize), sizeof(hashfilesize))!=sizeof(hashfilesize))
{
hashfilesize=little_endian((int64)-1);
out->Seek(0);
}
else
{
hashfilesize=little_endian(hashfilesize);
int64 hashdata_size = get_hashdata_size(hashfilesize);
if(out->Size()!=hashdata_size)
{
Server->Log(L"File \""+out_fn+L"\" has wrong size. Should="+convert(hashdata_size)+L" is="+convert(out->Size()), LL_ERROR);
assert(false);
}
out->Seek(hashdata_size);
}
if(!BackupServerPrepareHash::writeRepeatFreeSpace(out.get(), reinterpret_cast<char*>(&hashfilesize), sizeof(hashfilesize), cb))
{
Server->Log(L"Error writing file metadata to file \""+out_fn+L"\"", LL_ERROR);
return false;
}
return write_metadata(out.get(), cb, metadata);
}
bool is_metadata_only(IFile* hash_file)
{
int64 hashfilesize;
hash_file->Seek(0);
if(hash_file->Read(reinterpret_cast<char*>(&hashfilesize), sizeof(hashfilesize))!=sizeof(hashfilesize))
{
return false;
}
return hashfilesize==-1;
}
bool read_metadata(const std::wstring& in_fn, FileMetadata& metadata)
{
IFile* in = Server->openFile(os_file_prefix(in_fn));
if(!in)
{
Server->Log(L"Error reading file metadata from file \""+in_fn+L"\"", LL_DEBUG);
return false;
}
return read_metadata(in, metadata);
}
bool has_metadata( const std::wstring& in_fn, const FileMetadata& metadata )
{
FileMetadata r_metadata;
if(!read_metadata(in_fn, r_metadata))
{
return false;
}
return r_metadata==metadata;
}
std::wstring escape_metadata_fn( const std::wstring& fn )
{
if(fn.find(metadata_dir_fn)==0)
{
std::wstring num_str = fn.substr(sizeof(metadata_dir_fn)/sizeof(metadata_dir_fn[0])-1);
if(num_str.empty())
{
return fn+L"0";
}
size_t num_num = std::count_if(num_str.begin(), num_str.end(), static_cast<bool(*)(wchar_t)>(str_isnumber));
if(num_num!=num_str.size())
{
return fn;
}
return std::wstring(metadata_dir_fn)+convert(watoi64(num_str)+1);
}
else
{
return fn;
}
}
std::wstring unescape_metadata_fn( const std::wstring& fn )
{
if(fn.find(metadata_dir_fn)==0)
{
std::wstring num_str = fn.substr(sizeof(metadata_dir_fn)/sizeof(metadata_dir_fn[0])-1);
if(num_str.empty())
{
return fn;
}
size_t num_num = std::count_if(num_str.begin(), num_str.end(), static_cast<bool(*)(wchar_t)>(str_isnumber));
if(num_num!=num_str.size())
{
return fn;
}
int64 c = watoi64(num_str)-1;
if(c==0)
{
return metadata_dir_fn;
}
else
{
return std::wstring(metadata_dir_fn)+convert(c);
}
}
else
{
return fn;
}
}
void FileMetadata::serialize( CWData& data ) const
{
data.addString(file_permission_bits);
data.addInt64(last_modified);
data.addInt64(created);
}
bool FileMetadata::read( CRData& data )
{
bool ok=true;
ok &= data.getStr(&file_permission_bits);
ok &= data.getInt64(&last_modified);
ok &= data.getInt64(&created);
return ok;
}
bool FileMetadata::read( str_map& extra_params )
{
file_permission_bits = base64_decode_dash(wnarrow(extra_params[L"rpb"]));
last_modified = watoi64(extra_params[L"mod"]);
created = watoi64(extra_params[L"creat"]);
return true;
}
bool FileMetadata::bitSet( size_t id ) const
{
size_t bitmap_byte=(size_t)(id/8);
size_t bitmap_bit=id%8;
if(file_permission_bits.size()<bitmap_byte)
{
return false;
}
unsigned char b=static_cast<unsigned char>(file_permission_bits[bitmap_byte]);
return (b & (1<<(7-bitmap_bit)))>0;
}

57
urbackupserver/file_metadata.h Normal file
View File

@ -0,0 +1,57 @@
#pragma once
#include <string>
#include "../Interface/Types.h"
#include "../Interface/File.h"
#include "../stringtools.h"
class INotEnoughSpaceCallback;
class CWData;
class CRData;
class FileMetadata
{
public:
FileMetadata()
: last_modified(0), created(0)
{
}
std::string file_permission_bits;
int64 last_modified;
int64 created;
bool operator==(const FileMetadata& other)
{
return file_permission_bits==other.file_permission_bits
&& last_modified== other.last_modified
&& created == other.created;
}
bool bitSet(size_t id) const;
void serialize(CWData& data) const;
bool read(CRData& data);
bool read(str_map& extra_params);
};
bool write_file_metadata(const std::wstring& out_fn, INotEnoughSpaceCallback *cb, const FileMetadata& metadata);
bool is_metadata_only(IFile* hash_file);
bool read_metadata(const std::wstring& in_fn, FileMetadata& metadata);
bool has_metadata(const std::wstring& in_fn, const FileMetadata& metadata);
namespace
{
const wchar_t* metadata_dir_fn=L".dir_metadata";
}
std::wstring escape_metadata_fn(const std::wstring& fn);
std::wstring unescape_metadata_fn(const std::wstring& fn);

3
urbackupserver/fileclient/FileClient.cpp
View File

@ -20,6 +20,8 @@
#include "FileClient.h"
#include "../../fileservplugin/chunk_settings.h"
#include "../../common/data.h"
#include "../../stringtools.h"
@ -37,7 +39,6 @@
namespace
{
const std::string str_tmpdir="C:\\Windows\\Temp";
const _u64 c_checkpoint_dist=512*1024;
#ifndef _DEBUG
const unsigned int DISCOVERY_TIMEOUT=30000; //30sec
#else

12
urbackupserver/fileclient/FileClientChunked.cpp
View File

@ -10,11 +10,7 @@
#include <memory>
#include <algorithm>
#define VLOG(x) x
const unsigned int chunkhash_file_off=sizeof(_i64);
const unsigned int chunkhash_single_size=big_hash_size+small_hash_size*(c_checkpoint_dist/c_small_hash_dist);
const unsigned int c_reconnection_tries=30;
#define VLOG(x) x
unsigned int adler32(unsigned int adler, const char *buf, unsigned int len);
@ -121,6 +117,12 @@ _u32 FileClientChunked::GetFile(std::string remotefn)
hashfilesize = little_endian(hashfilesize);
if(hashfilesize<0)
{
Server->Log("Hashfile size wrong. Hashfile is damaged. Size is "+nconvert(hashfilesize), LL_ERROR);
return ERR_INT_ERROR;
}
if(patch_mode)
{
if(hashfilesize!=m_file->Size())

190
urbackupserver/filelist_utils.cpp Normal file
View File

@ -0,0 +1,190 @@
#include "filelist_utils.h"
#include "../Interface/Server.h"
#include "../stringtools.h"
void writeFileRepeat(IFile *f, const char *buf, size_t bsize)
{
_u32 written=0;
do
{
_u32 rc=f->Write(buf+written, (_u32)(bsize-written));
written+=rc;
if(rc==0)
{
Server->Log("Failed to write to file "+f->getFilename()+" retrying...", LL_WARNING);
Server->wait(10000);
}
}
while(written<bsize );
}
void writeFileRepeat(IFile *f, const std::string &str)
{
writeFileRepeat(f, str.c_str(), str.size());
}
std::string escapeListName( const std::string& listname )
{
std::string ret;
ret.reserve(listname.size());
for(size_t i=0;i<listname.size();++i)
{
if(listname[i]=='"')
{
ret+="\\\"";
}
else if(listname[i]=='\\')
{
ret+="\\\\";
}
else
{
ret+=listname[i];
}
}
return ret;
}
void writeFileItem(IFile* f, SFile cf)
{
if(cf.isdir)
{
writeFileRepeat(f, "d\""+escapeListName(Server->ConvertToUTF8(cf.name))+"\"\n");
}
else
{
writeFileRepeat(f, "f\""+escapeListName(Server->ConvertToUTF8(cf.name))+"\" "+nconvert(cf.size)+" "+nconvert(cf.last_modified)+"\n");
}
}
bool FileListParser::nextEntry( char ch, SFile &data, std::map<std::wstring, std::wstring>* extra )
{
switch(state)
{
case ParseState_Type:
if(ch=='f')
data.isdir=false;
else if(ch=='d')
data.isdir=true;
else
Server->Log("Error parsing file BackupServerGet::getNextEntry - 1", LL_ERROR);
state=ParseState_Quote;
break;
case ParseState_Quote:
//"
state=ParseState_Name;
break;
case ParseState_NameFinish:
if(ch!='"')
{
t_name.erase(t_name.size()-1,1);
data.name=Server->ConvertToUnicode(t_name);
t_name="";
if(data.isdir)
{
if(ch=='\n')
{
reset();
return true;
}
else
{
t_name+=ch;
state=ParseState_ExtraParams;
return false;
}
}
else
{
state=ParseState_Filesize;
}
}
//fallthrough
case ParseState_Name:
if(state==ParseState_Name && ch=='"')
{
state=ParseState_NameFinish;
}
else if(state==ParseState_Name && ch=='\\')
{
state=ParseState_NameEscape;
break;
}
else if(state==ParseState_NameFinish)
{
state=ParseState_Name;
}
t_name+=ch;
break;
case ParseState_NameEscape:
if(ch!='"' && ch!='\\')
{
t_name+='\\';
}
t_name+=ch;
state=ParseState_Name;
break;
case ParseState_Filesize:
if(ch!=' ')
{
t_name+=ch;
}
else
{
data.size=os_atoi64(t_name);
t_name="";
state=ParseState_ModifiedTime;
}
break;
case ParseState_ModifiedTime:
if(ch!='\n' && ch!='#')
{
t_name+=ch;
}
else
{
data.last_modified=os_atoi64(t_name);
if(ch=='\n')
{
reset();
return true;
}
else
{
t_name="";
state=ParseState_ExtraParams;
}
}
break;
case ParseState_ExtraParams:
if(ch!='\n')
{
t_name+=ch;
}
else
{
if(extra!=NULL)
{
ParseParamStrHttp(t_name, extra, false);
}
reset();
return true;
}
break;
}
return false;
}
void FileListParser::reset( void )
{
t_name="";
state=ParseState_Type;
}
FileListParser::FileListParser()
: state(ParseState_Type)
{
}

40
urbackupserver/filelist_utils.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#include "../Interface/File.h"
#include "../urbackupcommon/os_functions.h"
void writeFileRepeat(IFile *f, const char *buf, size_t bsize);
void writeFileRepeat(IFile *f, const std::string &str);
std::string escapeListName( const std::string& listname );
void writeFileItem(IFile* f, SFile cf);
class FileListParser
{
public:
FileListParser();
void reset(void);
bool nextEntry(char ch, SFile &data, std::map<std::wstring, std::wstring>* extra);
private:
enum ParseState
{
ParseState_Type,
ParseState_Quote,
ParseState_Name,
ParseState_NameEscape,
ParseState_NameFinish,
ParseState_Filesize,
ParseState_ModifiedTime,
ParseState_ExtraParams
};
ParseState state;
std::string t_name;
};

6
urbackupserver/server_archive.cpp
View File

@ -257,9 +257,15 @@ void ServerAutomaticArchive::copyArchiveSettings(int clientid)
q_insert_all->Reset();
}
IQuery *q_del_copied=db->Prepare("DELETE FROM settings_db.settings WHERE key='archive_settings_copied' AND clientid=?");
q_del_copied->Bind(clientid);
q_del_copied->Write();
q_del_copied->Reset();
IQuery *q_insert_copied=db->Prepare("INSERT INTO settings_db.settings (key, value, clientid) VALUES ('archive_settings_copied','true',?)");
q_insert_copied->Bind(clientid);
q_insert_copied->Write();
q_insert_copied->Reset();
}
bool ServerAutomaticArchive::isInArchiveWindow(const std::wstring &window_def)

2
urbackupserver/server_channel.cpp
View File

@ -302,6 +302,8 @@ int ServerChannelThread::constructCapabilities(void)
capa|=DONT_ALLOW_STARTING_INCR_IMAGE_BACKUPS;
if(!cs->allow_tray_exit)
capa|=DONT_ALLOW_EXIT_TRAY_ICON;
if(!cs->server_url.empty())
capa|=ALLOW_TOKEN_AUTHENTICATION;
return capa;
}

121
urbackupserver/server_download.cpp
View File

@ -6,11 +6,15 @@
#include "server_get.h"
#include "../stringtools.h"
#include "../common/data.h"
#include "file_metadata.h"
#include "server_settings.h"
#include "server_cleanup.h"
namespace
{
const unsigned int shadow_copy_timeout=30*60*1000;
const size_t max_queue_size = 5000;
const size_t minfreespace_min=50*1024*1024;
}
ServerDownloadThread::ServerDownloadThread( FileClient& fc, FileClientChunked* fc_chunked, bool with_hashes, const std::wstring& backuppath, const std::wstring& backuppath_hashes, const std::wstring& last_backuppath, const std::wstring& last_backuppath_complete, bool hashed_transfer, bool save_incomplete_file, int clientid,
@ -122,7 +126,7 @@ void ServerDownloadThread::operator()( void )
std::sort(download_nok_ids.begin(), download_nok_ids.end());
}
void ServerDownloadThread::addToQueueFull(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, bool at_front )
void ServerDownloadThread::addToQueueFull(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, const FileMetadata& metadata, bool at_front )
{
SQueueItem ni;
ni.id = id;
@ -135,6 +139,7 @@ void ServerDownloadThread::addToQueueFull(size_t id, const std::wstring &fn, con
ni.patch_dl_files.prepare_error=false;
ni.action = EQueueAction_Fileclient;
ni.predicted_filesize = predicted_filesize;
ni.metadata = metadata;
IScopedLock lock(mutex);
if(!at_front)
@ -154,7 +159,7 @@ void ServerDownloadThread::addToQueueFull(size_t id, const std::wstring &fn, con
}
void ServerDownloadThread::addToQueueChunked(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize )
void ServerDownloadThread::addToQueueChunked(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, const FileMetadata& metadata )
{
SQueueItem ni;
ni.id = id;
@ -167,6 +172,7 @@ void ServerDownloadThread::addToQueueChunked(size_t id, const std::wstring &fn,
ni.patch_dl_files.prepare_error=false;
ni.action = EQueueAction_Fileclient;
ni.predicted_filesize= predicted_filesize;
ni.metadata = metadata;
IScopedLock lock(mutex);
dl_queue.push_back(ni);
@ -271,14 +277,12 @@ bool ServerDownloadThread::load_file(SQueueItem todl)
if(hash_file)
{
std::wstring os_curr_path=BackupServerGet::convertToOSPathFromFileClient(todl.os_path+L"/"+todl.short_fn);
std::wstring os_curr_path=BackupServerGet::convertToOSPathFromFileClient(todl.os_path+L"/"+todl.short_fn);
std::wstring os_curr_hash_path=BackupServerGet::convertToOSPathFromFileClient(todl.os_path+L"/"+escape_metadata_fn(todl.short_fn));
std::wstring dstpath=backuppath+os_curr_path;
std::wstring hashpath;
std::wstring hashpath =backuppath_hashes+os_curr_hash_path;
std::wstring filepath_old;
if(with_hashes)
{
hashpath=backuppath_hashes+os_curr_path;
}
if( use_reflink && (!last_backuppath.empty() || !last_backuppath_complete.empty() ) )
{
std::wstring cfn_short=todl.os_path+L"/"+todl.short_fn;
@ -306,7 +310,7 @@ bool ServerDownloadThread::load_file(SQueueItem todl)
Server->destroy(file_old);
}
hashFile(dstpath, hashpath, fd, NULL, Server->ConvertToUTF8(filepath_old));
hashFile(dstpath, hashpath, fd, NULL, Server->ConvertToUTF8(filepath_old), todl.metadata);
}
return ret;
}
@ -325,7 +329,7 @@ bool ServerDownloadThread::load_file_patch(SQueueItem todl)
if(dlfiles.orig_file==NULL && full_dl)
{
addToQueueFull(todl.id, todl.fn, todl.short_fn, todl.curr_path, todl.os_path, todl.predicted_filesize, true);
addToQueueFull(todl.id, todl.fn, todl.short_fn, todl.curr_path, todl.os_path, todl.predicted_filesize, todl.metadata, true);
return true;
}
}
@ -335,73 +339,6 @@ bool ServerDownloadThread::load_file_patch(SQueueItem todl)
return false;
}
/*std::wstring cfn_short=todl.os_path+L"/"+todl.short_fn;
if(cfn_short[0]=='/')
cfn_short.erase(0,1);
std::wstring dstpath=backuppath+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
std::wstring hashpath=backuppath_hashes+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
std::wstring hashpath_old=last_backuppath+os_file_sep()+L".hashes"+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
std::wstring filepath_old=last_backuppath+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
std::auto_ptr<IFile> file_old(Server->openFile(os_file_prefix(filepath_old), MODE_READ));
if(file_old.get()==NULL)
{
if(!last_backuppath_complete.empty())
{
filepath_old=last_backuppath_complete+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
file_old.reset(Server->openFile(os_file_prefix(filepath_old), MODE_READ));
}
if(file_old.get()==NULL)
{
ServerLogger::Log(clientid, L"No old file for \""+todl.fn+L"\"", LL_DEBUG);
return load_file(todl);
}
hashpath_old=last_backuppath_complete+os_file_sep()+L".hashes"+os_file_sep()+BackupServerGet::convertToOSPathFromFileClient(cfn_short);
}
ServerLogger::Log(clientid, L"Loading file patch for \""+todl.fn+L"\"", LL_DEBUG);
IFile *pfd=BackupServerGet::getTemporaryFileRetry(use_tmpfiles, tmpfile_path, clientid);
if(pfd==NULL)
{
ServerLogger::Log(clientid, L"Error creating temporary file 'pfd' in load_file_patch", LL_ERROR);
return false;
}
ScopedDeleteFile pfd_destroy(pfd);
IFile *hash_tmp=BackupServerGet::getTemporaryFileRetry(use_tmpfiles, tmpfile_path, clientid);
if(hash_tmp==NULL)
{
ServerLogger::Log(clientid, L"Error creating temporary file 'hash_tmp' in load_file_patch", LL_ERROR);
return false;
}
ScopedDeleteFile hash_tmp_destroy(hash_tmp);
if(!server_token.empty())
{
cfn=widen(server_token)+L"|"+cfn;
}
std::auto_ptr<IFile> hashfile_old(Server->openFile(os_file_prefix(hashpath_old), MODE_READ));
bool delete_hashfile=false;
ScopedDeleteFile hashfile_old_destroy(NULL);
if( (hashfile_old.get()==NULL || hashfile_old->Size()==0 ) && file_old.get()!=NULL )
{
ServerLogger::Log(clientid, L"Hashes for file \""+filepath_old+L"\" not available. Calulating hashes...", LL_DEBUG);
hashfile_old.reset(BackupServerGet::getTemporaryFileRetry(use_tmpfiles, tmpfile_path, clientid));
if(hashfile_old.get()==NULL)
{
ServerLogger::Log(clientid, L"Error creating temporary file 'hashfile_old' in load_file_patch", LL_ERROR);
return false;
}
hashfile_old_destroy.reset(hashfile_old.get());
delete_hashfile=true;
BackupServerPrepareHash::build_chunk_hashs(file_old.get(), hashfile_old.get(), NULL, false, NULL, false);
hashfile_old->Seek(0);
} */
ServerLogger::Log(clientid, L"Loading file patch for \""+todl.fn+L"\"", LL_DEBUG);
@ -488,7 +425,8 @@ bool ServerDownloadThread::load_file_patch(SQueueItem todl)
pfd_destroy.release();
hash_tmp_destroy.release();
hashFile(dstpath, dlfiles.hashpath, dlfiles.patchfile, dlfiles.hashoutput, Server->ConvertToUTF8(dlfiles.filepath_old));
hashFile(dstpath, dlfiles.hashpath, dlfiles.patchfile, dlfiles.hashoutput,
Server->ConvertToUTF8(dlfiles.filepath_old), todl.metadata);
}
if(rc==ERR_TIMEOUT || rc==ERR_ERROR || rc==ERR_SOCKET_ERROR
@ -498,7 +436,7 @@ bool ServerDownloadThread::load_file_patch(SQueueItem todl)
return true;
}
void ServerDownloadThread::hashFile(std::wstring dstpath, std::wstring hashpath, IFile *fd, IFile *hashoutput, std::string old_file)
void ServerDownloadThread::hashFile(std::wstring dstpath, std::wstring hashpath, IFile *fd, IFile *hashoutput, std::string old_file, const FileMetadata& metadata)
{
int l_backup_id=backupid;
@ -518,6 +456,8 @@ void ServerDownloadThread::hashFile(std::wstring dstpath, std::wstring hashpath,
}
data.addString(old_file);
data.addChar(with_hashes?1:0);
metadata.serialize(data);
ServerLogger::Log(clientid, "GT: Loaded file \""+ExtractFileName(Server->ConvertToUTF8(dstpath))+"\"", LL_DEBUG);
@ -729,7 +669,10 @@ SPatchDownloadFiles ServerDownloadThread::preparePatchDownloadFiles( SQueueItem
std::auto_ptr<IFile> hashfile_old(Server->openFile(os_file_prefix(hashpath_old), MODE_READ));
dlfiles.delete_chunkhashes=false;
if( (hashfile_old.get()==NULL || hashfile_old->Size()==0 ) && file_old.get()!=NULL )
if( (hashfile_old.get()==NULL ||
hashfile_old->Size()==0 ||
is_metadata_only(hashfile_old.get()) )
&& file_old.get()!=NULL )
{
ServerLogger::Log(clientid, L"Hashes for file \""+filepath_old+L"\" not available. Calulating hashes...", LL_DEBUG);
hashfile_old.reset(BackupServerGet::getTemporaryFileRetry(use_tmpfiles, tmpfile_path, clientid));
@ -821,8 +764,13 @@ void ServerDownloadThread::unqueueFileChunked( const std::string& remotefn )
bool ServerDownloadThread::touch_file( SQueueItem todl )
{
std::wstring os_curr_path=BackupServerGet::convertToOSPathFromFileClient(todl.os_path+L"/"+todl.short_fn);
std::wstring dstpath=backuppath+os_curr_path;
std::wstring cfn_short=todl.os_path+L"/"+todl.short_fn;
if(cfn_short[0]=='/')
cfn_short.erase(0,1);
std::wstring os_curr_path=BackupServerGet::convertToOSPathFromFileClient(cfn_short);
std::wstring dstpath=backuppath+os_file_sep()+os_curr_path;
std::wstring hashpath=backuppath_hashes+os_file_sep()+os_curr_path;
ServerLogger::Log(clientid, L"GT: Touching file \""+dstpath+L"\"", LL_DEBUG);
@ -834,6 +782,13 @@ bool ServerDownloadThread::touch_file( SQueueItem todl )
max_ok_id=todl.id;
}
Server->destroy(f);
if(!write_file_metadata(hashpath, server_get, todl.metadata))
{
ServerLogger::Log(clientid, L"GT: Error writing file metadata to \""+hashpath+L"\"", LL_ERROR);
return false;
}
return true;
}
else
@ -842,4 +797,4 @@ bool ServerDownloadThread::touch_file( SQueueItem todl )
ServerLogger::Log(clientid, L"GT: Error creating file \""+dstpath+L"\"", LL_ERROR);
return false;
}
}
}

9
urbackupserver/server_download.h
View File

@ -11,6 +11,7 @@
#include "fileclient/FileClient.h"
#include "fileclient/FileClientChunked.h"
#include "server_get.h"
#include "file_metadata.h"
class FileClient;
@ -64,6 +65,7 @@ struct SQueueItem
bool queued;
EQueueAction action;
SPatchDownloadFiles patch_dl_files;
FileMetadata metadata;
};
class ServerDownloadThread : public IThread, public FileClient::QueueCallback, public FileClientChunked::QueueCallback
@ -73,13 +75,14 @@ public:
const std::wstring& clientname,
bool use_tmpfiles, const std::wstring& tmpfile_path, const std::string& server_token, bool use_reflink, int backupid, bool r_incremental, IPipe* hashpipe_prepare, BackupServerGet* server_get,
int filesrv_protocol_version);
~ServerDownloadThread();
void operator()(void);
void addToQueueFull(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, bool at_front=false);
void addToQueueFull(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, const FileMetadata& metadata, bool at_front=false);
void addToQueueChunked(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize);
void addToQueueChunked(size_t id, const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, _i64 predicted_filesize, const FileMetadata& metadata);
void addToQueueStartShadowcopy(const std::wstring& fn);
@ -99,7 +102,7 @@ public:
bool isOffline();
void hashFile(std::wstring dstpath, std::wstring hashpath, IFile *fd, IFile *hashoutput, std::string old_file);
void hashFile(std::wstring dstpath, std::wstring hashpath, IFile *fd, IFile *hashoutput, std::string old_file, const FileMetadata& metadata);
virtual bool getQueuedFileChunked(std::string& remotefn, IFile*& orig_file, IFile*& patchfile, IFile*& chunkhashes, IFile*& hashoutput, _i64& predicted_filesize);

312
urbackupserver/server_get.cpp
View File

@ -43,6 +43,7 @@
#include "server_hash_existing.h"
#include "server_dir_links.h"
#include "server.h"
#include "filelist_utils.h"
#include <algorithm>
#include <memory.h>
#include <time.h>
@ -156,64 +157,6 @@ BackupServerGet::~BackupServerGet(void)
Server->destroy(hash_existing_mutex);
}
namespace
{
void writeFileRepeat(IFile *f, const char *buf, size_t bsize)
{
_u32 written=0;
do
{
_u32 rc=f->Write(buf+written, (_u32)(bsize-written));
written+=rc;
if(rc==0)
{
Server->Log("Failed to write to file "+f->getFilename()+" retrying...", LL_WARNING);
Server->wait(10000);
}
}
while(written<bsize );
}
void writeFileRepeat(IFile *f, const std::string &str)
{
writeFileRepeat(f, str.c_str(), str.size());
}
std::string escapeListName( const std::string& listname )
{
std::string ret;
ret.reserve(listname.size());
for(size_t i=0;i<listname.size();++i)
{
if(listname[i]=='"')
{
ret+="\\\"";
}
else if(listname[i]=='\\')
{
ret+="\\\\";
}
else
{
ret+=listname[i];
}
}
return ret;
}
void writeFileItem(IFile* f, SFile cf)
{
if(cf.isdir)
{
writeFileRepeat(f, "d\""+escapeListName(Server->ConvertToUTF8(cf.name))+"\"\n");
}
else
{
writeFileRepeat(f, "f\""+escapeListName(Server->ConvertToUTF8(cf.name))+"\" "+nconvert(cf.size)+" "+nconvert(cf.last_modified)+"\n");
}
}
}
void BackupServerGet::init_mutex(void)
{
running_backup_mutex=Server->createMutex();
@ -1321,119 +1264,6 @@ void BackupServerGet::saveImageAssociation(int image_id, int assoc_id)
q_save_image_assoc->Reset();
}
bool BackupServerGet::getNextEntry(char ch, SFile &data, std::map<std::wstring, std::wstring>* extra)
{
switch(state)
{
case 0:
if(ch=='f')
data.isdir=false;
else if(ch=='d')
data.isdir=true;
else
ServerLogger::Log(clientid, "Error parsing file BackupServerGet::getNextEntry - 1", LL_ERROR);
state=1;
break;
case 1:
//"
state=2;
break;
case 3:
if(ch!='"')
{
t_name.erase(t_name.size()-1,1);
data.name=Server->ConvertToUnicode(t_name);
t_name="";
if(data.isdir)
{
resetEntryState();
return true;
}
else
state=4;
}
case 2:
if(state==2 && ch=='"')
{
state=3;
}
else if(state==2 && ch=='\\')
{
state=7;
break;
}
else if(state==3)
{
state=2;
}
t_name+=ch;
break;
case 7:
if(ch!='"' && ch!='\\')
{
t_name+='\\';
}
t_name+=ch;
state=2;
break;
case 4:
if(ch!=' ')
{
t_name+=ch;
}
else
{
data.size=os_atoi64(t_name);
t_name="";
state=5;
}
break;
case 5:
if(ch!='\n' && ch!='#')
{
t_name+=ch;
}
else
{
data.last_modified=os_atoi64(t_name);
if(ch=='\n')
{
resetEntryState();
return true;
}
else
{
t_name="";
state=6;
}
}
break;
case 6:
if(ch!='\n')
{
t_name+=ch;
}
else
{
if(extra!=NULL)
{
ParseParamStrHttp(t_name, extra, false);
}
resetEntryState();
return true;
}
break;
}
return false;
}
void BackupServerGet::resetEntryState(void)
{
t_name="";
state=0;
}
bool BackupServerGet::request_filelist_construct(bool full, bool resume, bool with_token, bool& no_backup_dirs, bool& connect_fail)
{
if(server_settings->getSettings()->end_to_end_file_backup_verification)
@ -1651,11 +1481,13 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
return false;
}
getTokenFile(fc, hashed_transfer);
backupid=createBackupSQL(0, clientid, backuppath_single, false, Server->getTimeMS()-indexing_start_time);
tmp->Seek(0);
resetEntryState();
FileListParser list_parser;
IFile *clientlist=Server->openFile("urbackup/clientlist_"+nconvert(clientid)+"_new.ub", MODE_WRITE);
@ -1729,9 +1561,12 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
for(size_t i=0;i<read;++i)
{
std::map<std::wstring, std::wstring> extra_params;
bool b=getNextEntry(buffer[i], cf, &extra_params);
bool b=list_parser.nextEntry(buffer[i], cf, &extra_params);
if(b)
{
FileMetadata metadata;
metadata.read(extra_params);
int64 ctime=Server->getTimeMS();
if(ctime-laststatsupdate>status_update_intervall)
{
@ -1777,12 +1612,18 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
c_has_error=true;
break;
}
if(with_hashes && !os_create_dir(os_file_prefix(backuppath_hashes+local_curr_os_path)))
if(!os_create_dir(os_file_prefix(backuppath_hashes+local_curr_os_path)))
{
ServerLogger::Log(clientid, L"Creating directory \""+backuppath_hashes+local_curr_os_path+L"\" failed.", LL_ERROR);
c_has_error=true;
break;
}
else if(!write_file_metadata(backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn, this, metadata))
{
ServerLogger::Log(clientid, L"Writing directory metadata to \""+backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn+L"\" failed.", LL_ERROR);
c_has_error=true;
break;
}
++depth;
if(depth==1)
{
@ -1790,7 +1631,6 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
t.erase(0,1);
ServerLogger::Log(clientid, L"Starting shadowcopy \""+t+L"\".", LL_INFO);
server_download->addToQueueStartShadowcopy(t);
Server->wait(10000);
}
}
else
@ -1813,7 +1653,8 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
std::map<std::wstring, std::wstring>::iterator hash_it=( (local_hash==NULL)?extra_params.end():extra_params.find(L"sha512") );
if( hash_it!=extra_params.end())
{
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, with_hashes, base64_decode_dash(wnarrow(hash_it->second)), cf.size, true))
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, base64_decode_dash(wnarrow(hash_it->second)), cf.size,
true, metadata))
{
file_ok=true;
linked_bytes+=cf.size;
@ -1825,7 +1666,8 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
}
if(!file_ok)
{
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1);
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
metadata);
}
}
@ -1882,12 +1724,12 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
tmp->Seek(0);
line = 0;
resetEntryState();
list_parser.reset();
while( (read=tmp->Read(buffer, 4096))>0 )
{
for(size_t i=0;i<read;++i)
{
bool b=getNextEntry(buffer[i], cf, NULL);
bool b=list_parser.nextEntry(buffer[i], cf, NULL);
if(b)
{
if(cf.isdir && line<max_line)
@ -1974,21 +1816,19 @@ bool BackupServerGet::doFullBackup(bool with_hashes, bool &disk_error, bool &log
return !r_done;
}
bool BackupServerGet::link_file(const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, bool with_hashes, const std::string& sha2, _i64 filesize, bool add_sql)
bool BackupServerGet::link_file(const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path,
const std::wstring &os_path, const std::string& sha2, _i64 filesize, bool add_sql, const FileMetadata& metadata)
{
std::wstring os_curr_path=convertToOSPathFromFileClient(os_path+L"/"+short_fn);
std::wstring os_curr_hash_path=convertToOSPathFromFileClient(os_path+L"/"+escape_metadata_fn(short_fn));
std::wstring dstpath=backuppath+os_curr_path;
std::wstring hashpath;
std::wstring hashpath=backuppath_hashes+os_curr_hash_path;
std::wstring filepath_old;
if(with_hashes)
{
hashpath=backuppath_hashes+os_curr_path;
}
bool tries_once;
std::wstring ff_last;
bool hardlink_limit;
bool ok=local_hash->findFileAndLink(dstpath, NULL, hashpath, sha2, true, filesize, std::string(), tries_once, ff_last, hardlink_limit);
bool ok=local_hash->findFileAndLink(dstpath, NULL, hashpath, sha2, filesize, std::string(), tries_once, ff_last, hardlink_limit, metadata);
if(ok && add_sql)
{
@ -2015,7 +1855,7 @@ _i64 BackupServerGet::getIncrementalSize(IFile *f, const std::vector<size_t> &di
{
f->Seek(0);
_i64 rsize=0;
resetEntryState();
FileListParser list_parser;
SFile cf;
bool indirchange=false;
size_t read;
@ -2035,7 +1875,7 @@ _i64 BackupServerGet::getIncrementalSize(IFile *f, const std::vector<size_t> &di
{
for(size_t i=0;i<read;++i)
{
bool b=getNextEntry(buffer[i], cf, NULL);
bool b=list_parser.nextEntry(buffer[i], cf, NULL);
if(b)
{
if(cf.isdir==true)
@ -2249,6 +2089,8 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
has_error=true;
return false;
}
getTokenFile(fc, hashed_transfer);
ServerLogger::Log(clientid, clientname+L" Starting incremental backup...", LL_DEBUG);
@ -2421,7 +2263,7 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
ServerLogger::Log(clientid, clientname+L": Linking unchanged and loading new files...", LL_DEBUG);
resetEntryState();
FileListParser list_parser;
bool c_has_error=false;
bool backup_stopped=false;
@ -2446,7 +2288,7 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
for(size_t i=0;i<read;++i)
{
std::map<std::wstring, std::wstring> extra_params;
bool b=getNextEntry(buffer[i], cf, &extra_params);
bool b=list_parser.nextEntry(buffer[i], cf, &extra_params);
if(b)
{
if(skip_dir_completely>0)
@ -2469,6 +2311,9 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
}
}
FileMetadata metadata;
metadata.read(extra_params);
int64 ctime=Server->getTimeMS();
if(ctime-laststatsupdate>status_update_intervall)
{
@ -2584,7 +2429,7 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
ServerLogger::Log(clientid, L"Directory \""+backuppath+local_curr_os_path+L"\" does already exist.", LL_WARNING);
}
}
if(with_hashes && !os_create_dir(os_file_prefix(backuppath_hashes+local_curr_os_path)))
if(!os_create_dir(os_file_prefix(backuppath_hashes+local_curr_os_path)))
{
if(!os_directory_exists(os_file_prefix(backuppath_hashes+local_curr_os_path)))
{
@ -2597,6 +2442,12 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
ServerLogger::Log(clientid, L"Directory \""+backuppath_hashes+local_curr_os_path+L"\" does already exist.", LL_WARNING);
}
}
else if(!write_file_metadata(backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn, this, metadata) )
{
ServerLogger::Log(clientid, L"Writing directory metadata to \""+backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn+L"\" failed.", LL_ERROR);
c_has_error=true;
break;
}
}
++depth;
if(depth==1)
@ -2647,7 +2498,8 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
bool f_ok=false;
if(!curr_sha2.empty())
{
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, with_hashes, curr_sha2 , cf.size, false))
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, curr_sha2 , cf.size, false,
metadata))
{
f_ok=true;
linked_bytes+=cf.size;
@ -2658,11 +2510,13 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
{
if(intra_file_diffs)
{
server_download->addToQueueChunked(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1);
server_download->addToQueueChunked(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
metadata);
}
else
{
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1);
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
metadata);
}
}
}
@ -2700,7 +2554,8 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
if(!curr_sha2.empty())
{
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, with_hashes, curr_sha2, cf.size, false))
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, curr_sha2, cf.size, false,
metadata))
{
f_ok=true;
copy_curr_file_entry=copy_file_entries;
@ -2713,11 +2568,13 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
{
if(intra_file_diffs)
{
server_download->addToQueueChunked(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1);
server_download->addToQueueChunked(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
metadata);
}
else
{
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1);
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
metadata);
}
}
}
@ -2820,12 +2677,12 @@ bool BackupServerGet::doIncrBackup(bool with_hashes, bool intra_file_diffs, bool
tmp->Seek(0);
line = 0;
resetEntryState();
list_parser.reset();
while( (read=tmp->Read(buffer, 4096))>0 )
{
for(size_t i=0;i<read;++i)
{
bool b=getNextEntry(buffer[i], cf, NULL);
bool b=list_parser.nextEntry(buffer[i], cf, NULL);
if(b)
{
if(cf.isdir)
@ -2995,7 +2852,7 @@ void BackupServerGet::waitForFileThreads(void)
bool BackupServerGet::deleteFilesInSnapshot(const std::string clientlist_fn, const std::vector<size_t> &deleted_ids, std::wstring snapshot_path, bool no_error)
{
resetEntryState();
FileListParser list_parser;
IFile *tmp=Server->openFile(clientlist_fn, MODE_READ);
if(tmp==NULL)
@ -3016,7 +2873,7 @@ bool BackupServerGet::deleteFilesInSnapshot(const std::string clientlist_fn, con
{
for(size_t i=0;i<read;++i)
{
if(getNextEntry(buffer[i], curr_file, NULL))
if(list_parser.nextEntry(buffer[i], curr_file, NULL))
{
if(curr_file.isdir)
{
@ -4617,7 +4474,19 @@ std::wstring BackupServerGet::fixFilenameForOS(const std::wstring& fn)
bool BackupServerGet::handle_not_enough_space(const std::wstring &path)
{
int64 free_space=os_free_space(os_file_prefix(server_settings->getSettings()->backupfolder));
int64 free_space=-1;
if(!path.empty())
{
free_space = os_free_space(os_file_prefix(path));
if(free_space==-1)
{
free_space = os_free_space(os_file_prefix(ExtractFilePath(path)));
}
}
if(free_space==-1)
{
free_space=os_free_space(os_file_prefix(server_settings->getSettings()->backupfolder));
}
if(free_space!=-1 && free_space<minfreespace_min)
{
Server->Log("No free space in backup folder. Free space="+PrettyPrintBytes(free_space)+" MinFreeSpace="+PrettyPrintBytes(minfreespace_min), LL_WARNING);
@ -4664,13 +4533,13 @@ bool BackupServerGet::verify_file_backup(IFile *fileentries)
size_t verified_files=0;
SFile cf;
fileentries->Seek(0);
resetEntryState();
FileListParser list_parser;
while( (read=fileentries->Read(buffer, 4096))>0 )
{
for(size_t i=0;i<read;++i)
{
std::map<std::wstring, std::wstring> extras;
bool b=getNextEntry(buffer[i], cf, &extras);
bool b=list_parser.nextEntry(buffer[i], cf, &extras);
if(b)
{
std::wstring cfn = fixFilenameForOS(cf.name);
@ -4981,6 +4850,12 @@ void BackupServerGet::run_script( std::wstring name, const std::wstring& params)
name = name + L".bat";
#endif
if(!FileExists(wnarrow(name)))
{
ServerLogger::Log(clientid, L"Script does not exist "+name, LL_DEBUG);
return;
}
name+=L" "+params;
name +=L" 2>&1";
@ -5025,3 +4900,32 @@ void BackupServerGet::run_script( std::wstring name, const std::wstring& params)
ServerLogger::Log(clientid, "Script output Line("+nconvert(i+1)+"): " + toks[i], rc!=0?LL_ERROR:LL_INFO);
}
}
void BackupServerGet::getTokenFile(FileClient &fc, bool hashed_transfer )
{
IFile *tokens_file=Server->openFile(os_file_prefix(backuppath_hashes+os_file_sep()+L".urbackup_tokens.properties"), MODE_WRITE);
if(tokens_file==NULL)
{
ServerLogger::Log(clientid, L"Error opening "+backuppath_hashes+os_file_sep()+L".urbackup_tokens.properties", LL_ERROR);
return;
}
_u32 rc=fc.GetFile("urbackup/tokens_"+server_token+".properties", tokens_file, hashed_transfer);
if(rc!=ERR_SUCCESS)
{
ServerLogger::Log(clientid, L"Error getting tokens file of "+clientname+L". Errorcode: "+widen(fc.getErrorString(rc))+L" ("+convert(rc)+L")", LL_INFO);
}
Server->destroy(tokens_file);
std::auto_ptr<ISettingsReader> urbackup_tokens(
Server->createFileSettingsReader(os_file_prefix(backuppath_hashes+os_file_sep()+L".urbackup_tokens.properties")));
std::string access_key;
if(urbackup_tokens->getValue("access_key", &access_key) &&
!access_key.empty() &&
access_key != server_settings->getSettings()->client_access_key )
{
backup_dao->updateOrInsertSetting(clientid, L"client_access_key", widen(access_key));
server_settings->update(true);
}
}

11
urbackupserver/server_get.h
View File

@ -100,9 +100,11 @@ private:
void notifyClientBackupSuccessfull(void);
bool request_filelist_construct(bool full, bool resume, bool with_token, bool& no_backup_dirs, bool& connect_fail);
bool link_file(const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, bool with_hashes, const std::string& sha2, _i64 filesize, bool add_sql);
bool link_file(const std::wstring &fn, const std::wstring &short_fn, const std::wstring &curr_path, const std::wstring &os_path, const std::string& sha2, _i64 filesize, bool add_sql, const FileMetadata& metadata);
bool doIncrBackup(bool with_hashes, bool intra_file_diffs, bool on_snapshot, bool use_directory_links, bool &disk_error, bool &log_backup, bool& r_incremental, bool& r_resumed);
void getTokenFile(FileClient &fc, bool hashed_transfer );
void calculateEtaFileBackup( int64 &last_eta_update, int64 ctime, FileClient &fc, FileClientChunked* fc_chunked, int64 linked_bytes, int64 &last_eta_received_bytes, double &eta_estimated_speed, _i64 files_size );
SBackup getLastIncremental(void);
@ -147,8 +149,7 @@ private:
std::wstring constructImagePath(const std::wstring &letter, bool compress);
bool constructBackupPath(bool with_hashes, bool on_snapshot, bool create_fs);
void resetEntryState(void);
bool getNextEntry(char ch, SFile &data, std::map<std::wstring, std::wstring>* extra);
static std::string remLeadingZeros(std::string t);
bool updateCapabilities(void);
@ -244,10 +245,6 @@ private:
IQuery *q_format_unixtime;
IQuery *q_get_last_incremental_complete;
int state;
std::string t_name;
int link_logcnt;
IPipe *hashpipe;

88
urbackupserver/server_hash.cpp
View File

@ -33,6 +33,8 @@
#include "create_files_cache.h"
#include <algorithm>
#include <memory.h>
#include "file_metadata.h"
#include <assert.h>
const size_t freespace_mod=50*1024*1024; //50 MB
const size_t BUFFER_SIZE=64*1024; //64KB
@ -245,11 +247,15 @@ void BackupServerHash::operator()(void)
std::string hashoutput_fn;
rd.getStr(&hashoutput_fn);
bool diff_file=!hashoutput_fn.empty();
std::string old_file_fn;
rd.getStr(&old_file_fn);
char with_hashes_c;
rd.getChar(&with_hashes_c);
FileMetadata metadata;
metadata.read(rd);
IFile *tf=Server->openFile(os_file_prefix(Server->ConvertToUnicode(temp_fn)), MODE_READ_SEQUENTIAL);
if(tf==NULL)
@ -260,10 +266,10 @@ void BackupServerHash::operator()(void)
else
{
addFile(backupid, incremental, tf, Server->ConvertToUnicode(tfn), Server->ConvertToUnicode(hashpath), sha2,
diff_file, old_file_fn, hashoutput_fn);
old_file_fn, hashoutput_fn, with_hashes_c==1, metadata);
}
if(diff_file)
if(!hashoutput_fn.empty())
{
Server->deleteFile(hashoutput_fn);
}
@ -384,7 +390,8 @@ void BackupServerHash::deleteFileSQL(const std::string &pHash, const std::wstrin
}
bool BackupServerHash::findFileAndLink(const std::wstring &tfn, IFile *tf, std::wstring& hash_fn, const std::string &sha2,
bool diff_file, _i64 t_filesize, const std::string &hashoutput_fn, bool &tries_once, std::wstring &ff_last, bool &hardlink_limit)
_i64 t_filesize, const std::string &hashoutput_fn, bool &tries_once, std::wstring &ff_last, bool &hardlink_limit,
const FileMetadata& metadata)
{
hardlink_limit=false;
@ -454,7 +461,14 @@ bool BackupServerHash::findFileAndLink(const std::wstring &tfn, IFile *tf, std::
}
else
{
if(!hash_fn.empty() && !f_hashpath.empty())
if(hash_fn.find(L"config.log")!=std::string::npos)
{
int a4=4;
}
assert(!hash_fn.empty());
bool do_write_metadata=false;
if(!f_hashpath.empty() &&
has_metadata(f_hashpath, metadata) )
{
b=os_create_hardlink(os_file_prefix(hash_fn), os_file_prefix(f_hashpath), use_snapshots, NULL);
if(!b)
@ -463,33 +477,30 @@ bool BackupServerHash::findFileAndLink(const std::wstring &tfn, IFile *tf, std::
if(ctf==NULL)
{
ServerLogger::Log(clientid, "HT: Hardlinking hash file failed (File doesn't exist)", LL_DEBUG);
if(!diff_file)
if(!hashoutput_fn.empty())
{
if(tf!=NULL && !createChunkHashes(tf, hash_fn))
ServerLogger::Log(clientid, "HT: Creating chunk hash file failed", LL_ERROR);
}
else
{
if(!hashoutput_fn.empty())
IFile *src=openFileRetry(Server->ConvertToUnicode(hashoutput_fn), MODE_READ);
if(src!=NULL)
{
IFile *src=openFileRetry(Server->ConvertToUnicode(hashoutput_fn), MODE_READ);
if(src!=NULL)
if(!copyFile(src, hash_fn))
{
if(!copyFile(src, hash_fn))
{
ServerLogger::Log(clientid, "Error copying hashoutput to destination -1", LL_ERROR);
has_error=true;
hash_fn.clear();
}
Server->destroy(src);
}
else
{
ServerLogger::Log(clientid, "HT: Error opening hashoutput", LL_ERROR);
ServerLogger::Log(clientid, "Error copying hashoutput to destination -1", LL_ERROR);
has_error=true;
hash_fn.clear();
}
Server->destroy(src);
}
else
{
ServerLogger::Log(clientid, "HT: Error opening hashoutput", LL_ERROR);
has_error=true;
hash_fn.clear();
}
}
else
{
do_write_metadata=true;
}
}
else
@ -504,6 +515,20 @@ bool BackupServerHash::findFileAndLink(const std::wstring &tfn, IFile *tf, std::
}
}
}
else
{
do_write_metadata=true;
}
if(do_write_metadata)
{
if(!write_file_metadata(hash_fn, this, metadata))
{
ServerLogger::Log(clientid, "Error writing file metadata -1", LL_ERROR);
has_error=true;
}
}
copy=false;
break;
}
@ -525,7 +550,8 @@ bool BackupServerHash::findFileAndLink(const std::wstring &tfn, IFile *tf, std::
}
void BackupServerHash::addFile(int backupid, char incremental, IFile *tf, const std::wstring &tfn,
std::wstring hash_fn, const std::string &sha2, bool diff_file, const std::string &orig_fn, const std::string &hashoutput_fn)
std::wstring hash_fn, const std::string &sha2, const std::string &orig_fn, const std::string &hashoutput_fn,
bool with_hashes, const FileMetadata& metadata)
{
_i64 t_filesize=tf->Size();
@ -533,7 +559,7 @@ void BackupServerHash::addFile(int backupid, char incremental, IFile *tf, const
bool tries_once;
std::wstring ff_last;
bool hardlink_limit;
if(findFileAndLink(tfn, tf, hash_fn, sha2, diff_file, t_filesize,hashoutput_fn, tries_once, ff_last, hardlink_limit))
if(findFileAndLink(tfn, tf, hash_fn, sha2, t_filesize, hashoutput_fn, tries_once, ff_last, hardlink_limit, metadata))
{
ServerLogger::Log(clientid, L"HT: Linked file: \""+tfn+L"\"", LL_DEBUG);
copy=false;
@ -626,11 +652,11 @@ void BackupServerHash::addFile(int backupid, char incremental, IFile *tf, const
else
{
bool r;
if(!diff_file)
if(hashoutput_fn.empty())
{
if(!use_reflink || orig_fn.empty())
{
if(!hash_fn.empty())
if(!hash_fn.empty() && with_hashes)
{
if(use_tmpfiles)
{
@ -657,7 +683,7 @@ void BackupServerHash::addFile(int backupid, char incremental, IFile *tf, const
}
else
{
if(!hash_fn.empty())
if(!hash_fn.empty() && with_hashes)
{
r=replaceFileWithHashoutput(tf, tfn, hash_fn, Server->ConvertToUnicode(orig_fn));
}

9
urbackupserver/server_hash.h
View File

@ -12,6 +12,9 @@
#include "dao/ServerBackupDao.h"
#include <vector>
class FileMetadata;
struct STmpFile
{
STmpFile(int backupid, std::wstring fp, std::wstring hashpath)
@ -51,7 +54,8 @@ public:
void setupDatabase(void);
void deinitDatabase(void);
bool findFileAndLink(const std::wstring &tfn, IFile *tf, std::wstring& hash_fn, const std::string &sha2, bool diff_file, _i64 t_filesize, const std::string &hashoutput_fn, bool &tries_once, std::wstring &ff_last, bool &hardlink_limit);
bool findFileAndLink(const std::wstring &tfn, IFile *tf, std::wstring& hash_fn, const std::string &sha2, _i64 t_filesize,
const std::string &hashoutput_fn, bool &tries_once, std::wstring &ff_last, bool &hardlink_limit, const FileMetadata& metadata);
void addFileSQL(int backupid, char incremental, const std::wstring &fp, const std::wstring &hash_path, const std::string &shahash, _i64 filesize, _i64 rsize);
@ -60,7 +64,8 @@ public:
private:
void prepareSQL(void);
void addFile(int backupid, char incremental, IFile *tf, const std::wstring &tfn,
std::wstring hash_fn, const std::string &sha2, bool diff_file, const std::string &orig_fn, const std::string &hashoutput_fn);
std::wstring hash_fn, const std::string &sha2, const std::string &orig_fn, const std::string &hashoutput_fn,
bool with_hashes, const FileMetadata& metadata);
std::wstring findFileHash(const std::string &pHash, _i64 filesize, int &backupid, std::wstring &hashpath, bool& cache_hit);
std::wstring findFileHashTmp(const std::string &pHash, _i64 filesize, int &backupid, std::wstring &hashpath);
bool copyFile(IFile *tf, const std::wstring &dest);

9
urbackupserver/server_prepare_hash.cpp
View File

@ -28,6 +28,7 @@
#include "../fileservplugin/chunk_settings.h"
#include "../md5.h"
#include <memory.h>
#include "file_metadata.h"
BackupServerPrepareHash::BackupServerPrepareHash(IPipe *pPipe, IPipe *pOutput, int pClientid)
{
@ -92,6 +93,12 @@ void BackupServerPrepareHash::operator()(void)
std::string old_file_fn;
rd.getStr(&old_file_fn);
char with_hashes_c;
rd.getChar(&with_hashes_c);
FileMetadata metadata;
metadata.read(rd);
IFile *tf=Server->openFile(os_file_prefix(Server->ConvertToUnicode(temp_fn)), MODE_READ);
IFile *old_file=NULL;
if(diff_file)
@ -143,6 +150,8 @@ void BackupServerPrepareHash::operator()(void)
data.addString(h);
data.addString(hashoutput_fn);
data.addString(old_file_fn);
data.addChar(with_hashes_c);
metadata.serialize(data);
output->Write(data.getDataPtr(), data.getDataSize() );
}

3
urbackupserver/server_settings.cpp
View File

@ -319,6 +319,7 @@ void ServerSettings::readSettingsDefault(void)
settings->trust_client_hashes=(settings_default->getValue("trust_client_hashes", "true")=="true");
settings->internet_connect_always=(settings_default->getValue("internet_connect_always", "false")=="true");
settings->show_server_updates=(settings_default->getValue("show_server_updates", "true")=="true");
settings->server_url=settings_default->getValue("server_url", "");
}
void ServerSettings::readSettingsClient(void)
@ -334,6 +335,8 @@ void ServerSettings::readSettingsClient(void)
settings->internet_authkey=generateRandomAuthKey();
}
settings->client_access_key = settings_client->getValue("client_access_key", std::string());
stmp=settings_client->getValue("client_overwrite", "");
if(!stmp.empty())
settings->client_overwrite=(stmp=="true");

2
urbackupserver/server_settings.h
View File

@ -92,6 +92,8 @@ struct SSettings
bool trust_client_hashes;
bool internet_connect_always;
bool show_server_updates;
std::string server_url;
std::string client_access_key;
};
struct STimeSpan

416
urbackupserver/serverinterface/backups.cpp
View File

@ -18,6 +18,13 @@
#include "action_header.h"
#include "../../urbackupcommon/os_functions.h"
#include "../file_metadata.h"
#include "../../Interface/SettingsReader.h"
#include "../../cryptoplugin/ICryptoFactory.h"
#include "../server_settings.h"
#include "backups.h"
extern ICryptoFactory *crypto_fak;
std::string constructFilter(const std::vector<int> &clientid, std::string key)
{
@ -32,7 +39,8 @@ std::string constructFilter(const std::vector<int> &clientid, std::string key)
return clientf;
}
bool create_zip_to_output(const std::wstring& foldername, const std::wstring& filter);
bool create_zip_to_output(const std::wstring& foldername, const std::wstring& hashfoldername, const std::wstring& filter, bool token_authentication,
const std::vector<SToken> &backup_tokens, const std::vector<std::string> &tokens, bool skip_hashes);
namespace
{
@ -65,7 +73,8 @@ namespace
}
}
bool sendZip(Helper& helper, const std::wstring& foldername, const std::wstring& filter)
bool sendZip(Helper& helper, const std::wstring& foldername, const std::wstring& hashfoldername, const std::wstring& filter, bool token_authentication,
const std::vector<SToken>& backup_tokens, const std::vector<std::string>& tokens, bool skip_hashes)
{
std::wstring zipname=ExtractFileName(foldername)+L".zip";
@ -74,8 +83,239 @@ namespace
Server->addHeader(tid, "Content-Disposition: attachment; filename=\""+Server->ConvertToUTF8(zipname)+"\"");
helper.releaseAll();
return create_zip_to_output(foldername, filter);
return create_zip_to_output(foldername, hashfoldername, filter, token_authentication,
backup_tokens, tokens, skip_hashes);
}
std::vector<FileMetadata> getMetadata(std::wstring dir, const std::vector<SFile>& files, bool skip_hashes)
{
std::vector<FileMetadata> ret;
ret.resize(files.size());
if(dir.empty() || dir[dir.size()-1]!=os_file_sep()[0])
{
dir+=os_file_sep();
}
for(size_t i=0;i<files.size();++i)
{
if(skip_hashes && files[i].name==L".hashes")
continue;
std::wstring metadata_fn;
if(files[i].isdir)
metadata_fn = dir+escape_metadata_fn(files[i].name)+os_file_sep()+metadata_dir_fn;
else
metadata_fn = dir+escape_metadata_fn(files[i].name);
if(!read_metadata(metadata_fn, ret[i]) )
{
Server->Log(L"Error reading metadata of file "+dir+os_file_sep()+files[i].name, LL_ERROR);
}
}
return ret;
}
int getClientid(IDatabase* db, const std::wstring& clientname)
{
IQuery* q=db->Prepare("SELECT id FROM clients WHERE name=?");
q->Bind(clientname);
db_results res=q->Read();
q->Reset();
if(!res.empty())
{
return watoi(res[0][L"id"]);
}
return -1;
}
std::wstring getClientname(IDatabase* db, int clientid)
{
IQuery *q=db->Prepare("SELECT name FROM clients WHERE id=?");
q->Bind(clientid);
db_results res=q->Read();
q->Reset();
if(!res.empty())
{
return res[0][L"name"];
}
else
{
return std::wstring();
}
}
std::wstring getBackupFolder(IDatabase* db)
{
IQuery* q=db->Prepare("SELECT value FROM settings_db.settings WHERE key='backupfolder'");
db_results res_bf=q->Read();
q->Reset();
if(!res_bf.empty() )
{
return res_bf[0][L"value"];
}
else
{
return std::wstring();
}
}
bool checkBackupTokens(const std::string& fileaccesstokens, const std::wstring& backupfolder, const std::wstring& clientname, const std::wstring& path)
{
std::vector<std::string> tokens;
Tokenize(fileaccesstokens, tokens, ";");
STokens backup_tokens = readTokens(backupfolder, clientname, path);
if(backup_tokens.tokens.empty())
{
return false;
}
for(size_t i=0;i<backup_tokens.tokens.size();++i)
{
for(size_t j=0;j<tokens.size();++j)
{
if(backup_tokens.tokens[i].token.empty())
continue;
if(backup_tokens.tokens[i].token==tokens[j])
{
return true;
}
}
}
return false;
}
bool checkFileToken(const std::string& fileaccesstokens, const std::wstring& backupfolder, const std::wstring& clientname, const std::wstring& path, const std::wstring& filemetadatapath)
{
std::vector<std::string> tokens;
Tokenize(fileaccesstokens, tokens, ";");
STokens backup_tokens = readTokens(backupfolder, clientname, path);
if(backup_tokens.tokens.empty())
{
return false;
}
FileMetadata metadata;
if(!read_metadata(filemetadatapath, metadata))
{
return false;
}
return checkFileToken(backup_tokens.tokens, tokens, metadata);
}
std::string decryptTokens(IDatabase* db, str_map GET)
{
if(crypto_fak==NULL)
{
return std::string();
}
int clientid;
str_map::iterator iter_clientname =GET.find(L"clientname");
if(iter_clientname!=GET.end())
{
clientid = getClientid(db, iter_clientname->second);
}
else
{
str_map::iterator iter_clientid = GET.find(L"clientid");
if(iter_clientid!=GET.end())
{
clientid = watoi(iter_clientid->second);
}
else
{
return std::string();
}
}
if(clientid==-1)
{
return std::string();
}
ServerSettings server_settings(db, clientid);
std::string client_key = server_settings.getSettings()->client_access_key;
size_t i=0;
str_map::iterator iter;
do
{
iter = GET.find(L"tokens"+convert(i));
if(iter!=GET.end())
{
std::string decry = crypto_fak->decryptAuthenticatedAES(base64_decode_dash(wnarrow(iter->second)), client_key);
if(!decry.empty())
{
return decry;
}
}
} while (iter!=GET.end());
return std::string();
}
}
STokens readTokens(const std::wstring& backupfolder, const std::wstring& clientname, const std::wstring& path)
{
if(backupfolder.empty() || clientname.empty() || path.empty())
{
return STokens();
}
std::auto_ptr<ISettingsReader> backup_tokens(Server->createFileSettingsReader(backupfolder+os_file_sep()+clientname+os_file_sep()+path+os_file_sep()+L".hashes"+os_file_sep()+L".urbackup_tokens.properties"));
if(!backup_tokens.get())
{
return STokens();
}
std::string ids_str = backup_tokens->getValue("ids", "");
std::vector<std::string> ids;
Tokenize(ids_str, ids, ",");
std::vector<SToken> ret;
for(size_t i=0;i<ids.size();++i)
{
SToken token = { watoi64(widen(ids[i])),
backup_tokens->getValue(ids[i]+"."+"username", ""),
backup_tokens->getValue(ids[i]+"."+"token", "") };
ret.push_back(token);
}
STokens tokens = { backup_tokens->getValue("access_key", ""),
ret };
return tokens;
}
bool checkFileToken( const std::vector<SToken> &backup_tokens, const std::vector<std::string> &tokens, const FileMetadata &metadata )
{
for(size_t i=0;i<backup_tokens.size();++i)
{
for(size_t j=0;j<tokens.size();++j)
{
if(backup_tokens[i].token.empty())
continue;
if(backup_tokens[i].token==tokens[j] &&
metadata.bitSet(backup_tokens[i].id))
{
return true;
}
}
}
return false;
}
ACTION_IMPL(backups)
@ -83,7 +323,50 @@ ACTION_IMPL(backups)
Helper helper(tid, &GET, &PARAMS);
JSON::Object ret;
SUser *session=helper.getSession();
if(session!=NULL && session->id==-1) return;
bool has_tokens = GET.find(L"tokens0")!=GET.end();
bool token_authentication=false;
std::string fileaccesstokens;
if( (session==NULL || session->id==-1) && has_tokens)
{
token_authentication=true;
fileaccesstokens = decryptTokens(helper.getDatabase(), GET);
if(fileaccesstokens.empty())
{
return;
}
else
{
std::wstring ses=helper.generateSession(L"anonymous");
ret.set("session", ses);
GET[L"ses"]=ses;
helper.update(tid, &GET, &PARAMS);
if(helper.getSession())
{
helper.getSession()->mStr[L"fileaccesstokens"]=widen(fileaccesstokens);
}
}
}
else if(session!=NULL && session->id==-1)
{
fileaccesstokens = wnarrow(session->mStr[L"fileaccesstokens"]);
if(!fileaccesstokens.empty())
{
token_authentication=true;
}
else
{
return;
}
}
if(token_authentication)
{
ret.set("token_authentication", true);
}
std::wstring sa=GET[L"sa"];
std::string rights=helper.getRights("browse_backups");
std::string archive_rights=helper.getRights("manual_archive");
@ -94,10 +377,14 @@ ACTION_IMPL(backups)
sa=L"backups";
GET[L"clientid"]=convert(clientid[0]);
}
if(session!=NULL && rights!="none")
if(token_authentication && sa.empty())
{
sa=L"backups";
}
if( (session!=NULL && rights!="none" ) || token_authentication)
{
IDatabase *db=helper.getDatabase();
if(sa.empty())
if(sa.empty() && !token_authentication)
{
std::string qstr="SELECT id, name, strftime('"+helper.getTimeFormatString()+"', lastbackup, 'localtime') AS lastbackup FROM clients";
if(!clientid.empty()) qstr+=" WHERE "+constructFilter(clientid, "id");
@ -119,11 +406,31 @@ ACTION_IMPL(backups)
}
else if(sa==L"backups")
{
int t_clientid=watoi(GET[L"clientid"]);
bool r_ok=helper.hasRights(t_clientid, rights, clientid);
bool archive_ok=helper.hasRights(t_clientid, archive_rights, clientid_archive);
int t_clientid;
std::wstring clientname;
if(r_ok)
if(token_authentication && GET.find(L"clientid")==GET.end())
{
clientname=GET[L"clientname"];
t_clientid = getClientid(helper.getDatabase(), clientname);
if(t_clientid==-1)
{
ret.set("error", "2");
helper.Write(ret.get(false));
return;
}
}
else
{
t_clientid=watoi(GET[L"clientid"]);
clientname = getClientname(helper.getDatabase(), t_clientid);
}
bool r_ok=token_authentication?false:helper.hasRights(t_clientid, rights, clientid);
bool archive_ok=token_authentication?false:helper.hasRights(t_clientid, archive_rights, clientid_archive);
if(r_ok || token_authentication)
{
if(archive_ok)
{
@ -141,12 +448,19 @@ ACTION_IMPL(backups)
}
}
IQuery *q=db->Prepare("SELECT id, strftime('"+helper.getTimeFormatString()+"', backuptime, 'localtime') AS t_backuptime, incremental, size_bytes, archived, archive_timeout FROM backups WHERE complete=1 AND done=1 AND clientid=? ORDER BY backuptime DESC");
std::wstring backupfolder = getBackupFolder(helper.getDatabase());
IQuery *q=db->Prepare("SELECT id, strftime('"+helper.getTimeFormatString()+"', backuptime, 'localtime') AS t_backuptime, incremental, size_bytes, archived, archive_timeout, path FROM backups WHERE complete=1 AND done=1 AND clientid=? ORDER BY backuptime DESC");
q->Bind(t_clientid);
db_results res=q->Read();
JSON::Array backups;
for(size_t i=0;i<res.size();++i)
{
if(token_authentication && !checkBackupTokens(fileaccesstokens, backupfolder, clientname, res[i][L"path"]) )
{
continue;
}
JSON::Object obj;
obj.set("id", watoi(res[i][L"id"]));
obj.set("backuptime", res[i][L"t_backuptime"]);
@ -168,15 +482,8 @@ ACTION_IMPL(backups)
ret.set("backups", backups);
ret.set("can_archive", archive_ok);
q=db->Prepare("SELECT name FROM clients WHERE id=?");
q->Bind(t_clientid);
res=q->Read();
q->Reset();
if(!res.empty())
{
ret.set("clientname", res[0][L"name"]);
ret.set("clientid", t_clientid);
}
ret.set("clientname", clientname);
ret.set("clientid", t_clientid);
}
else
{
@ -186,7 +493,8 @@ ACTION_IMPL(backups)
else if(sa==L"files" || sa==L"filesdl" || sa==L"zipdl" )
{
int t_clientid=watoi(GET[L"clientid"]);
bool r_ok=helper.hasRights(t_clientid, rights, clientid);
bool r_ok = token_authentication ? true :
helper.hasRights(t_clientid, rights, clientid);
if(r_ok)
{
@ -197,7 +505,9 @@ ACTION_IMPL(backups)
Tokenize(u_path, t_path, L"/");
for(size_t i=0;i<t_path.size();++i)
{
if(!t_path[i].empty() && t_path[i]!=L" " && t_path[i]!=L"." && t_path[i]!=L".." && t_path[i].find(L"/")==std::string::npos && t_path[i].find(L"\\")==std::string::npos )
if(!t_path[i].empty() && t_path[i]!=L" " && t_path[i]!=L"." && t_path[i]!=L".."
&& t_path[i].find(L"/")==std::string::npos
&& t_path[i].find(L"\\")==std::string::npos )
{
path+=UnescapeSQLString(t_path[i])+os_file_sep();
}
@ -209,49 +519,58 @@ ACTION_IMPL(backups)
path.erase(path.size()-1, 1);
}
IQuery *q=db->Prepare("SELECT name FROM clients WHERE id=?");
q->Bind(t_clientid);
db_results res=q->Read();
q->Reset();
q=db->Prepare("SELECT value FROM settings_db.settings WHERE key='backupfolder'");
db_results res_bf=q->Read();
q->Reset();
if(!res.empty() && !res_bf.empty() )
std::wstring clientname=getClientname(helper.getDatabase(), t_clientid);
std::wstring backupfolder=getBackupFolder(db);
if(!clientname.empty() && !backupfolder.empty() )
{
std::wstring backupfolder=res_bf[0][L"value"];
std::wstring clientname=res[0][L"name"];
ret.set("clientname", clientname);
ret.set("clientid", t_clientid);
ret.set("backupid", backupid);
ret.set("path", u_path);
q=db->Prepare("SELECT path,strftime('"+helper.getTimeFormatString()+"', backuptime, 'localtime') AS backuptime FROM backups WHERE id=?");
IQuery* q=db->Prepare("SELECT path,strftime('"+helper.getTimeFormatString()+"', backuptime, 'localtime') AS backuptime FROM backups WHERE id=?");
q->Bind(backupid);
res=q->Read();
db_results res=q->Read();
q->Reset();
if(!res.empty())
if(!res.empty() &&
(!token_authentication || checkBackupTokens(fileaccesstokens, backupfolder, clientname, res[0][L"path"])) )
{
std::wstring backuppath=res[0][L"path"];
ret.set("backuptime", res[0][L"backuptime"]);
std::wstring currdir=backupfolder+os_file_sep()+clientname+os_file_sep()+backuppath+(path.empty()?L"":(os_file_sep()+path));
std::wstring curr_metadata_dir=backupfolder+os_file_sep()+clientname+os_file_sep()+backuppath+os_file_sep()+L".hashes"+(path.empty()?L"":(os_file_sep()+path));
if(sa==L"filesdl")
{
sendFile(helper, currdir);
if(!token_authentication || checkFileToken(fileaccesstokens, backupfolder, clientname, backuppath, curr_metadata_dir))
{
sendFile(helper, currdir);
}
return;
}
else if(sa==L"zipdl")
STokens backup_tokens;
std::vector<std::string> tokens;
if(token_authentication)
{
sendZip(helper, currdir, GET[L"filter"]);
backup_tokens = readTokens(backupfolder, clientname, backuppath);
Tokenize(fileaccesstokens, tokens, ";");
}
if(sa==L"zipdl")
{
sendZip(helper, currdir, curr_metadata_dir, GET[L"filter"], token_authentication, backup_tokens.tokens, tokens, path.empty());
return;
}
std::vector<SFile> tfiles=getFiles(os_file_prefix(currdir), NULL, true, false);
std::vector<FileMetadata> tmetadata=getMetadata(curr_metadata_dir, tfiles, path.empty());
JSON::Array files;
for(size_t i=0;i<tfiles.size();++i)
@ -261,10 +580,18 @@ ACTION_IMPL(backups)
if(path.empty() && tfiles[i].name==L".hashes")
continue;
if(token_authentication &&
!checkFileToken(backup_tokens.tokens, tokens, tmetadata[i]))
{
continue;
}
JSON::Object obj;
obj.set("name", tfiles[i].name);
obj.set("dir", tfiles[i].isdir);
obj.set("size", tfiles[i].size);
obj.set("mod", tmetadata[i].last_modified);
obj.set("creat", tmetadata[i].created);
files.add(obj);
}
}
@ -272,10 +599,21 @@ ACTION_IMPL(backups)
{
if(!tfiles[i].isdir)
{
if(path.empty() && tfiles[i].name==L".urbackup_tokens.properties")
continue;
if(token_authentication &&
!checkFileToken(backup_tokens.tokens, tokens, tmetadata[i]))
{
continue;
}
JSON::Object obj;
obj.set("name", tfiles[i].name);
obj.set("dir", tfiles[i].isdir);
obj.set("size", tfiles[i].size);
obj.set("mod", tmetadata[i].last_modified);
obj.set("creat", tmetadata[i].created);
files.add(obj);
}
}

20
urbackupserver/serverinterface/backups.h Normal file
View File

@ -0,0 +1,20 @@
#pragma once
#include "../file_metadata.h"
struct SToken
{
int64 id;
std::string username;
std::string token;
};
struct STokens
{
std::string access_key;
std::vector<SToken> tokens;
};
STokens readTokens(const std::wstring& backupfolder, const std::wstring& clientname, const std::wstring& path);
bool checkFileToken( const std::vector<SToken> &backup_tokens, const std::vector<std::string> &tokens, const FileMetadata &metadata );

32
urbackupserver/serverinterface/create_zip.cpp
View File

@ -1,6 +1,7 @@
#include "action_header.h"
#include "../../urbackupcommon/os_functions.h"
#include "../../Interface/File.h"
#include "backups.h"
#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES
#include "../../common/miniz.c"
@ -255,7 +256,8 @@ bool miniz_init(mz_zip_archive *pZip, MiniZFileInfo* fileInfo)
return true;
}
bool add_dir(mz_zip_archive& zip_archive, const std::wstring& archivefoldername, const std::wstring& foldername, const std::wstring& filter)
bool add_dir(mz_zip_archive& zip_archive, const std::wstring& archivefoldername, const std::wstring& foldername, const std::wstring& hashfoldername, const std::wstring& filter,
bool token_authentication, const std::vector<SToken> &backup_tokens, const std::vector<std::string> &tokens, bool skip_hashes)
{
bool has_error=false;
const std::vector<SFile> files = getFiles(foldername, &has_error, true, false);
@ -267,12 +269,32 @@ bool add_dir(mz_zip_archive& zip_archive, const std::wstring& archivefoldername,
{
const SFile& file=files[i];
if(skip_hashes
&& file.name==L".hashes")
{
continue;
}
std::wstring archivename = archivefoldername + (archivefoldername.empty()?L"":L"/") + file.name;
std::wstring metadataname = hashfoldername + os_file_sep() + escape_metadata_fn(file.name);
std::wstring filename = foldername + os_file_sep() + file.name;
if(!filter.empty() && archivename!=filter)
continue;
if(file.isdir)
{
metadataname+=os_file_sep()+metadata_dir_fn;
}
FileMetadata metadata;
if(token_authentication &&
( !read_metadata(metadataname, metadata) ||
!checkFileToken(backup_tokens, tokens, metadata) ) )
{
continue;
}
mz_bool rc;
if(file.isdir)
{
@ -291,7 +313,8 @@ bool add_dir(mz_zip_archive& zip_archive, const std::wstring& archivefoldername,
if(file.isdir)
{
add_dir(zip_archive, archivename, filename, filter);
add_dir(zip_archive, archivename, filename, hashfoldername + os_file_sep() + file.name, filter,
token_authentication, backup_tokens, tokens, false);
}
}
@ -300,7 +323,8 @@ bool add_dir(mz_zip_archive& zip_archive, const std::wstring& archivefoldername,
}
bool create_zip_to_output(const std::wstring& foldername, const std::wstring& filter)
bool create_zip_to_output(const std::wstring& foldername, const std::wstring& hashfoldername, const std::wstring& filter, bool token_authentication,
const std::vector<SToken> &backup_tokens, const std::vector<std::string> &tokens, bool skip_hashes)
{
mz_zip_archive zip_archive;
memset(&zip_archive, 0, sizeof(zip_archive));
@ -315,7 +339,7 @@ bool create_zip_to_output(const std::wstring& foldername, const std::wstring& fi
return false;
}
if(!add_dir(zip_archive, L"", foldername, filter))
if(!add_dir(zip_archive, L"", foldername, hashfoldername, filter, token_authentication, backup_tokens, tokens, skip_hashes))
{
Server->Log("Error while adding files and folders to ZIP archive", LL_ERROR);
return false;

7
urbackupserver/serverinterface/livelog.cpp
View File

@ -59,6 +59,13 @@ ACTION_IMPL(livelog)
SUser *session=helper.getSession();
if(session!=NULL && session->id==-1) return;
if(session==NULL)
{
JSON::Object ret;
ret.set("error", true);
helper.Write(ret.get(false));
return;
}
int clientid=0;
std::wstring s_clientid=GET[L"clientid"];

1
urbackupserver/serverinterface/settings.cpp
View File

@ -185,6 +185,7 @@ void getGeneralSettings(JSON::Object& obj, IDatabase *db, ServerSettings &settin
SET_SETTING(use_incremental_symlinks);
SET_SETTING(trust_client_hashes);
SET_SETTING(show_server_updates);
SET_SETTING(server_url);
#undef SET_SETTING
}

6
urbackupserver/serverinterface/start_backup.cpp
View File

@ -42,7 +42,11 @@ ACTION_IMPL(start_backup)
std::wstring s_start_client=GET[L"start_client"];
std::vector<int> start_client;
std::wstring start_type=GET[L"start_type"];
if(!s_start_client.empty() && helper.getRights("start_backup")=="all")
SUser *session=helper.getSession();
if(session!=NULL && session->id==-1) return;
if(session!=NULL && !s_start_client.empty() && helper.getRights("start_backup")=="all")
{
std::vector<SStatus> client_status=ServerStatus::getStatus();

5
urbackupserver/urbackupserver.vcxproj
View File

@ -278,6 +278,8 @@
<ClCompile Include="FileCache.cpp" />
<ClCompile Include="fileclient\FileClientChunked.cpp" />
<ClCompile Include="filedownload.cpp" />
<ClCompile Include="filelist_utils.cpp" />
<ClCompile Include="file_metadata.cpp" />
<ClCompile Include="InternetServiceConnector.cpp" />
<ClCompile Include="lmdb\mdb.c" />
<ClCompile Include="lmdb\midl.c" />
@ -363,6 +365,8 @@
<ClInclude Include="fileclient\packet_ids.h" />
<ClInclude Include="fileclient\socket_header.h" />
<ClInclude Include="filedownload.h" />
<ClInclude Include="filelist_utils.h" />
<ClInclude Include="file_metadata.h" />
<ClInclude Include="InternetServiceConnector.h" />
<ClInclude Include="lmdb\lmdb.h" />
<ClInclude Include="lmdb\midl.h" />
@ -370,6 +374,7 @@
<ClInclude Include="server.h" />
<ClInclude Include="serverinterface\actions.h" />
<ClInclude Include="serverinterface\action_header.h" />
<ClInclude Include="serverinterface\backups.h" />
<ClInclude Include="serverinterface\helper.h" />
<ClInclude Include="serverinterface\login.h" />
<ClInclude Include="serverinterface\rights.h" />

15
urbackupserver/urbackupserver.vcxproj.filters
View File

@ -264,6 +264,12 @@
<ClCompile Include="server_hash_existing.cpp">
<Filter>Quelldateien</Filter>
</ClCompile>
<ClCompile Include="filelist_utils.cpp">
<Filter>Quelldateien</Filter>
</ClCompile>
<ClCompile Include="file_metadata.cpp">
<Filter>Quelldateien</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="action_header.h">
@ -470,5 +476,14 @@
<ClInclude Include="server_hash_existing.h">
<Filter>Headerdateien</Filter>
</ClInclude>
<ClInclude Include="filelist_utils.h">
<Filter>Headerdateien</Filter>
</ClInclude>
<ClInclude Include="file_metadata.h">
<Filter>Headerdateien</Filter>
</ClInclude>
<ClInclude Include="serverinterface\backups.h">
<Filter>serverinterface</Filter>
</ClInclude>
</ItemGroup>
</Project>

1
urbackupserver/www/index.htm
View File

@ -20,6 +20,7 @@
<script language="JavaScript" src="jqplot.barRenderer.min.js"></script>
<script language="JavaScript" src="jqplot.dateAxisRenderer.min.js"></script>
<script language="JavaScript" src="jqplot.highlighter.min.js"></script>
<script language="JavaScript" src="jquery.ba-bbq.min.js"></script>
<script language="JavaScript" src="templates.js"></script>
<script language="JavaScript" src="urbackup_functions.js"></script>
<script language="JavaScript" src="default_user_rights.js"></script>

18
urbackupserver/www/jquery.ba-bbq.min.js vendored Normal file
View File

@ -0,0 +1,18 @@
/*
* jQuery BBQ: Back Button & Query Library - v1.2.1 - 2/17/2010
* http://benalman.com/projects/jquery-bbq-plugin/
*
* Copyright (c) 2010 "Cowboy" Ben Alman
* Dual licensed under the MIT and GPL licenses.
* http://benalman.com/about/license/
*/
(function($,p){var i,m=Array.prototype.slice,r=decodeURIComponent,a=$.param,c,l,v,b=$.bbq=$.bbq||{},q,u,j,e=$.event.special,d="hashchange",A="querystring",D="fragment",y="elemUrlAttr",g="location",k="href",t="src",x=/^.*\?|#.*$/g,w=/^.*\#/,h,C={};function E(F){return typeof F==="string"}function B(G){var F=m.call(arguments,1);return function(){return G.apply(this,F.concat(m.call(arguments)))}}function n(F){return F.replace(/^[^#]*#?(.*)$/,"$1")}function o(F){return F.replace(/(?:^[^?#]*\?([^#]*).*$)?.*/,"$1")}function f(H,M,F,I,G){var O,L,K,N,J;if(I!==i){K=F.match(H?/^([^#]*)\#?(.*)$/:/^([^#?]*)\??([^#]*)(#?.*)/);J=K[3]||"";if(G===2&&E(I)){L=I.replace(H?w:x,"")}else{N=l(K[2]);I=E(I)?l[H?D:A](I):I;L=G===2?I:G===1?$.extend({},I,N):$.extend({},N,I);L=a(L);if(H){L=L.replace(h,r)}}O=K[1]+(H?"#":L||!K[1]?"?":"")+L+J}else{O=M(F!==i?F:p[g][k])}return O}a[A]=B(f,0,o);a[D]=c=B(f,1,n);c.noEscape=function(G){G=G||"";var F=$.map(G.split(""),encodeURIComponent);h=new RegExp(F.join("|"),"g")};c.noEscape(",/");$.deparam=l=function(I,F){var H={},G={"true":!0,"false":!1,"null":null};$.each(I.replace(/\+/g," ").split("&"),function(L,Q){var K=Q.split("="),P=r(K[0]),J,O=H,M=0,R=P.split("]["),N=R.length-1;if(/\[/.test(R[0])&&/\]$/.test(R[N])){R[N]=R[N].replace(/\]$/,"");R=R.shift().split("[").concat(R);N=R.length-1}else{N=0}if(K.length===2){J=r(K[1]);if(F){J=J&&!isNaN(J)?+J:J==="undefined"?i:G[J]!==i?G[J]:J}if(N){for(;M<=N;M++){P=R[M]===""?O.length:R[M];O=O[P]=M<N?O[P]||(R[M+1]&&isNaN(R[M+1])?{}:[]):J}}else{if($.isArray(H[P])){H[P].push(J)}else{if(H[P]!==i){H[P]=[H[P],J]}else{H[P]=J}}}}else{if(P){H[P]=F?i:""}}});return H};function z(H,F,G){if(F===i||typeof F==="boolean"){G=F;F=a[H?D:A]()}else{F=E(F)?F.replace(H?w:x,""):F}return l(F,G)}l[A]=B(z,0);l[D]=v=B(z,1);$[y]||($[y]=function(F){return $.extend(C,F)})({a:k,base:k,iframe:t,img:t,input:t,form:"action",link:k,script:t});j=$[y];function s(I,G,H,F){if(!E(H)&&typeof H!=="object"){F=H;H=G;G=i}return this.each(function(){var L=$(this),J=G||j()[(this.nodeName||"").toLowerCase()]||"",K=J&&L.attr(J)||"";L.attr(J,a[I](K,H,F))})}$.fn[A]=B(s,A);$.fn[D]=B(s,D);b.pushState=q=function(I,F){if(E(I)&&/^#/.test(I)&&F===i){F=2}var H=I!==i,G=c(p[g][k],H?I:{},H?F:2);p[g][k]=G+(/#/.test(G)?"":"#")};b.getState=u=function(F,G){return F===i||typeof F==="boolean"?v(F):v(G)[F]};b.removeState=function(F){var G={};if(F!==i){G=u();$.each($.isArray(F)?F:arguments,function(I,H){delete G[H]})}q(G,2)};e[d]=$.extend(e[d],{add:function(F){var H;function G(J){var I=J[D]=c();J.getState=function(K,L){return K===i||typeof K==="boolean"?l(I,K):l(I,L)[K]};H.apply(this,arguments)}if($.isFunction(F)){H=F;return G}else{H=F.handler;F.handler=G}}})})(jQuery,this);
/*
* jQuery hashchange event - v1.2 - 2/11/2010
* http://benalman.com/projects/jquery-hashchange-plugin/
*
* Copyright (c) 2010 "Cowboy" Ben Alman
* Dual licensed under the MIT and GPL licenses.
* http://benalman.com/about/license/
*/
(function($,i,b){var j,k=$.event.special,c="location",d="hashchange",l="href",f=$.browser,g=document.documentMode,h=f.msie&&(g===b||g<8),e="on"+d in i&&!h;function a(m){m=m||i[c][l];return m.replace(/^[^#]*#?(.*)$/,"$1")}$[d+"Delay"]=100;k[d]=$.extend(k[d],{setup:function(){if(e){return false}$(j.start)},teardown:function(){if(e){return false}$(j.stop)}});j=(function(){var m={},r,n,o,q;function p(){o=q=function(s){return s};if(h){n=$('<iframe src="javascript:0"/>').hide().insertAfter("body")[0].contentWindow;q=function(){return a(n.document[c][l])};o=function(u,s){if(u!==s){var t=n.document;t.open().close();t[c].hash="#"+u}};o(a())}}m.start=function(){if(r){return}var t=a();o||p();(function s(){var v=a(),u=q(t);if(v!==t){o(t=v,u);$(i).trigger(d)}else{if(u!==t){i[c][l]=i[c][l].replace(/#.*/,"")+"#"+u}}r=setTimeout(s,$[d+"Delay"])})()};m.stop=function(){if(!n){r&&clearTimeout(r);r=0}};return m})()})(jQuery,this);

14
urbackupserver/www/templates.js
View File

File diff suppressed because one or more lines are too long

2
urbackupserver/www/templates/backups_backups.htm
View File

@ -1,4 +1,4 @@
<a href="javascript: show_backups1()">{tClients}</a> > <strong>{clientname}</strong><br /><br />
{?show_client_breadcrumb}<a href="javascript: show_backups1()">{tClients}</a> > {/show_client_breadcrumb}<strong>{clientname}</strong><br /><br />
<table cellspacing="0" cellpadding="0">
<tr>

2
urbackupserver/www/templates/backups_files.htm
View File

@ -1,4 +1,4 @@
<a href="javascript: show_backups1()">{tClients}</a> > <a href="javascript: tabMouseClickClients({clientid})">{clientname}</a> > {cpath|s}<br /><br />
{?show_client_breadcrumb}<a href="javascript: show_backups1()">{tClients}</a> > {/show_client_breadcrumb}<a href="javascript: tabMouseClickClients({clientid})">{clientname}</a> > {cpath|s}<br /><br />
<table cellspacing="0" cellpadding="0">
<tr>

4
urbackupserver/www/templates/settings_general.htm
View File

@ -7,6 +7,10 @@
<td><input type="text" size="40" id="backupfolder" value="{backupfolder|s}"/> </td>
</tr>
<tr>
<td>{tServer URL}:</td>
<td><input type="text" size="40" id="server_url" value="{server_url|s}"/> </td>
</tr>
<tr>
<td>{tDo not do image backups}:</td>
<td><input type="checkbox" id="no_images" value="true" {no_images}/></td>
</tr>

35
urbackupserver/www/urbackup.js
View File

@ -92,6 +92,7 @@ function startup()
if(i+1<g.languages.length)
available_langs+=",";
}
new getJSON("login", available_langs, try_anonymous_login);
}
@ -213,23 +214,37 @@ function try_anonymous_login(data)
return;
}
if(data.success)
params = $.deparam(window.location.hash);
if(params && params.fileaccesstokens)
{
g.session=data.session;
build_main_nav();
show_status1();
file_access(params);
}
else
{
var ndata=dustRender("login");
if(g.data_f!=ndata)
{
if(data.success)
{
I('data_f').innerHTML=ndata;
g.data_f=ndata;
g.session=data.session;
build_main_nav();
show_status1();
}
else
{
var ndata=dustRender("login");
if(g.data_f!=ndata)
{
I('data_f').innerHTML=ndata;
g.data_f=ndata;
}
I('username').focus();
}
I('username').focus();
}
}
function file_access(params)
{
new getJSON("backups", available_langs, try_anonymous_login);
}
function startLoading()
{
if(g.loading)