mirror of
https://github.com/uroni/urbackup_backend.git
synced 2025-10-26 11:36:50 +00:00
558 lines
16 KiB
C++
558 lines
16 KiB
C++
/*************************************************************************
|
|
* UrBackup - Client/Server backup system
|
|
* Copyright (C) 2011-2014 Martin Raiber
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
**************************************************************************/
|
|
|
|
#include "FullFileBackup.h"
|
|
#include "database.h"
|
|
#include <vector>
|
|
#include "../Interface/Server.h"
|
|
#include "dao/ServerBackupDao.h"
|
|
#include "ClientMain.h"
|
|
#include "server_log.h"
|
|
#include "fileclient/FileClient.h"
|
|
#include "filelist_utils.h"
|
|
#include "server_running.h"
|
|
#include "ServerDownloadThread.h"
|
|
#include "file_metadata.h"
|
|
#include <stack>
|
|
|
|
extern std::string server_identity;
|
|
extern std::string server_token;
|
|
|
|
|
|
FullFileBackup::FullFileBackup( ClientMain* client_main, int clientid, std::wstring clientname, LogAction log_action, int group, bool use_tmpfiles, std::wstring tmpfile_path, bool use_reflink, bool use_snapshots )
|
|
: FileBackup(client_main, clientid, clientname, log_action, false, group, use_tmpfiles, tmpfile_path, use_reflink, use_snapshots)
|
|
{
|
|
|
|
}
|
|
|
|
|
|
SBackup FullFileBackup::getLastFullDurations( void )
|
|
{
|
|
std::vector<ServerBackupDao::SDuration> durations =
|
|
backup_dao->getLastFullDurations(clientid);
|
|
|
|
ServerBackupDao::SDuration duration = interpolateDurations(durations);
|
|
|
|
SBackup b;
|
|
|
|
b.indexing_time_ms = duration.indexing_time_ms;
|
|
b.backup_time_ms = duration.duration*1000;
|
|
|
|
return b;
|
|
}
|
|
|
|
bool FullFileBackup::doFileBackup()
|
|
{
|
|
ServerLogger::Log(clientid, "Starting full file backup...", LL_INFO);
|
|
|
|
SBackup last_backup_info = getLastFullDurations();
|
|
|
|
status.eta_ms = last_backup_info.backup_time_ms + last_backup_info.indexing_time_ms;
|
|
ServerStatus::setServerStatus(status, true);
|
|
|
|
int64 indexing_start_time = Server->getTimeMS();
|
|
|
|
bool no_backup_dirs=false;
|
|
bool connect_fail=false;
|
|
bool b=request_filelist_construct(true, false, group, true, no_backup_dirs, connect_fail);
|
|
if(!b)
|
|
{
|
|
has_early_error=true;
|
|
|
|
if(no_backup_dirs || connect_fail)
|
|
{
|
|
log_backup=false;
|
|
}
|
|
else
|
|
{
|
|
log_backup=true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool hashed_transfer=true;
|
|
bool save_incomplete_files=false;
|
|
|
|
if(client_main->isOnInternetConnection())
|
|
{
|
|
if(server_settings->getSettings()->internet_full_file_transfer_mode=="raw")
|
|
hashed_transfer=false;
|
|
if(server_settings->getSettings()->internet_incr_file_transfer_mode=="blockhash")
|
|
save_incomplete_files=true;
|
|
}
|
|
else
|
|
{
|
|
if(server_settings->getSettings()->local_full_file_transfer_mode=="raw")
|
|
hashed_transfer=false;
|
|
if(server_settings->getSettings()->local_incr_file_transfer_mode=="blockhash")
|
|
save_incomplete_files=true;
|
|
}
|
|
|
|
if(hashed_transfer)
|
|
{
|
|
ServerLogger::Log(clientid, clientname+L": Doing backup with hashed transfer...", LL_DEBUG);
|
|
}
|
|
else
|
|
{
|
|
ServerLogger::Log(clientid, clientname+L": Doing backup without hashed transfer...", LL_DEBUG);
|
|
}
|
|
std::string identity = client_main->getSessionIdentity().empty()?server_identity:client_main->getSessionIdentity();
|
|
FileClient fc(false, identity, client_main->getProtocolVersions().filesrv_protocol_version,
|
|
client_main->isOnInternetConnection(), client_main, use_tmpfiles?NULL:client_main);
|
|
_u32 rc=client_main->getClientFilesrvConnection(&fc, server_settings.get(), 10000);
|
|
if(rc!=ERR_CONNECTED)
|
|
{
|
|
ServerLogger::Log(clientid, L"Full Backup of "+clientname+L" failed - CONNECT error", LL_ERROR);
|
|
has_early_error=true;
|
|
log_backup=false;
|
|
return false;
|
|
}
|
|
|
|
IFile *tmp=ClientMain::getTemporaryFileRetry(use_tmpfiles, tmpfile_path, clientid);
|
|
if(tmp==NULL)
|
|
{
|
|
ServerLogger::Log(clientid, L"Error creating temporary file in ::doFullBackup", LL_ERROR);
|
|
return false;
|
|
}
|
|
|
|
ServerLogger::Log(clientid, clientname+L": Loading file list...", LL_INFO);
|
|
|
|
int64 full_backup_starttime=Server->getTimeMS();
|
|
|
|
rc=fc.GetFile(group>0?("urbackup/filelist_"+nconvert(group)+".ub"):"urbackup/filelist.ub", tmp, hashed_transfer);
|
|
if(rc!=ERR_SUCCESS)
|
|
{
|
|
ServerLogger::Log(clientid, L"Error getting filelist of "+clientname+L". Errorcode: "+widen(fc.getErrorString(rc))+L" ("+convert(rc)+L")", LL_ERROR);
|
|
has_early_error=true;
|
|
return false;
|
|
}
|
|
|
|
getTokenFile(fc, hashed_transfer);
|
|
|
|
backup_dao->newFileBackup(0, clientid, backuppath_single, 0, Server->getTimeMS()-indexing_start_time, group);
|
|
backupid = static_cast<int>(db->getLastInsertID());
|
|
|
|
tmp->Seek(0);
|
|
|
|
FileListParser list_parser;
|
|
|
|
IFile *clientlist=Server->openFile(clientlistName(group, true), MODE_WRITE);
|
|
|
|
if(clientlist==NULL )
|
|
{
|
|
ServerLogger::Log(clientid, L"Error creating clientlist for client "+clientname, LL_ERROR);
|
|
has_early_error=true;
|
|
return false;
|
|
}
|
|
|
|
if(ServerStatus::isBackupStopped(clientname))
|
|
{
|
|
ServerLogger::Log(clientid, L"Server admin stopped backup. -1", LL_ERROR);
|
|
has_early_error=true;
|
|
return false;
|
|
}
|
|
|
|
_i64 filelist_size=tmp->Size();
|
|
|
|
char buffer[4096];
|
|
_u32 read;
|
|
std::wstring curr_path;
|
|
std::wstring curr_os_path;
|
|
SFile cf;
|
|
int depth=0;
|
|
bool r_done=false;
|
|
int64 laststatsupdate=0;
|
|
int64 last_eta_update=0;
|
|
int64 last_eta_received_bytes=0;
|
|
double eta_estimated_speed=0;
|
|
ServerStatus::setServerStatus(status, true);
|
|
ServerRunningUpdater *running_updater=new ServerRunningUpdater(backupid, false);
|
|
Server->getThreadPool()->execute(running_updater);
|
|
|
|
ServerLogger::Log(clientid, clientname+L": Started loading files...", LL_INFO);
|
|
|
|
std::wstring last_backuppath;
|
|
std::wstring last_backuppath_complete;
|
|
std::auto_ptr<ServerDownloadThread> server_download(new ServerDownloadThread(fc, NULL, backuppath,
|
|
backuppath_hashes, last_backuppath, last_backuppath_complete,
|
|
hashed_transfer, save_incomplete_files, clientid, clientname,
|
|
use_tmpfiles, tmpfile_path, server_token, use_reflink,
|
|
backupid, false, hashpipe_prepare, client_main, client_main->getProtocolVersions().filesrv_protocol_version, 0));
|
|
|
|
bool queue_downloads = client_main->getProtocolVersions().filesrv_protocol_version>2;
|
|
|
|
THREADPOOL_TICKET server_download_ticket =
|
|
Server->getThreadPool()->execute(server_download.get());
|
|
|
|
std::vector<size_t> diffs;
|
|
_i64 files_size=getIncrementalSize(tmp, diffs, true);
|
|
fc.resetReceivedDataBytes();
|
|
tmp->Seek(0);
|
|
|
|
size_t line = 0;
|
|
int64 linked_bytes = 0;
|
|
|
|
size_t max_ok_id=0;
|
|
|
|
bool c_has_error=false;
|
|
bool is_offline=false;
|
|
bool script_dir=false;
|
|
|
|
std::stack<FileMetadata> dir_metadata;
|
|
|
|
while( (read=tmp->Read(buffer, 4096))>0 && r_done==false && c_has_error==false)
|
|
{
|
|
if(ServerStatus::isBackupStopped(clientname))
|
|
{
|
|
r_done=true;
|
|
ServerLogger::Log(clientid, L"Server admin stopped backup.", LL_ERROR);
|
|
server_download->queueSkip();
|
|
break;
|
|
}
|
|
|
|
for(size_t i=0;i<read;++i)
|
|
{
|
|
std::map<std::wstring, std::wstring> extra_params;
|
|
bool b=list_parser.nextEntry(buffer[i], cf, &extra_params);
|
|
if(b)
|
|
{
|
|
FileMetadata metadata;
|
|
metadata.read(extra_params);
|
|
|
|
int64 ctime=Server->getTimeMS();
|
|
if(ctime-laststatsupdate>status_update_intervall)
|
|
{
|
|
laststatsupdate=ctime;
|
|
if(files_size==0)
|
|
{
|
|
status.pcdone=100;
|
|
}
|
|
else
|
|
{
|
|
status.pcdone=(std::min)(100,(int)(((float)fc.getReceivedDataBytes() + linked_bytes)/((float)files_size/100.f)+0.5f));
|
|
}
|
|
status.hashqueuesize=(_u32)hashpipe->getNumElements();
|
|
status.prepare_hashqueuesize=(_u32)hashpipe_prepare->getNumElements();
|
|
ServerStatus::setServerStatus(status, true);
|
|
}
|
|
|
|
if(ctime-last_eta_update>eta_update_intervall)
|
|
{
|
|
calculateEtaFileBackup(last_eta_update, ctime, fc, NULL, linked_bytes, last_eta_received_bytes, eta_estimated_speed, files_size);
|
|
}
|
|
|
|
if(server_download->isOffline())
|
|
{
|
|
ServerLogger::Log(clientid, L"Client "+clientname+L" went offline.", LL_ERROR);
|
|
is_offline = true;
|
|
r_done=true;
|
|
break;
|
|
}
|
|
|
|
std::wstring osspecific_name=fixFilenameForOS(cf.name);
|
|
if(cf.isdir)
|
|
{
|
|
if(cf.name!=L"..")
|
|
{
|
|
dir_metadata.push(metadata);
|
|
|
|
curr_path+=L"/"+cf.name;
|
|
curr_os_path+=L"/"+osspecific_name;
|
|
std::wstring local_curr_os_path=convertToOSPathFromFileClient(curr_os_path);
|
|
|
|
if(!os_create_dir(os_file_prefix(backuppath+local_curr_os_path)))
|
|
{
|
|
ServerLogger::Log(clientid, L"Creating directory \""+backuppath+local_curr_os_path+L"\" failed. " + widen(systemErrorInfo()), LL_ERROR);
|
|
c_has_error=true;
|
|
break;
|
|
}
|
|
else if(metadata.exist)
|
|
{
|
|
if(!os_set_file_time(os_file_prefix(backuppath+local_curr_os_path),
|
|
metadata.created, metadata.last_modified))
|
|
{
|
|
ServerLogger::Log(clientid, L"Setting last modified and created time of directory \""+backuppath+local_curr_os_path+L"\" failed. " + widen(systemErrorInfo()), LL_WARNING);
|
|
}
|
|
}
|
|
if(!os_create_dir(os_file_prefix(backuppath_hashes+local_curr_os_path)))
|
|
{
|
|
ServerLogger::Log(clientid, L"Creating directory \""+backuppath_hashes+local_curr_os_path+L"\" failed. " + widen(systemErrorInfo()), LL_ERROR);
|
|
c_has_error=true;
|
|
break;
|
|
}
|
|
else if(metadata.exist && !write_file_metadata(backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn, client_main, metadata))
|
|
{
|
|
ServerLogger::Log(clientid, L"Writing directory metadata to \""+backuppath_hashes+local_curr_os_path+os_file_sep()+metadata_dir_fn+L"\" failed.", LL_ERROR);
|
|
c_has_error=true;
|
|
break;
|
|
}
|
|
++depth;
|
|
if(depth==1)
|
|
{
|
|
std::wstring t=curr_path;
|
|
t.erase(0,1);
|
|
if(t==L"urbackup_backup_scripts")
|
|
{
|
|
script_dir=true;
|
|
}
|
|
else
|
|
{
|
|
ServerLogger::Log(clientid, L"Starting shadowcopy \""+t+L"\".", LL_DEBUG);
|
|
server_download->addToQueueStartShadowcopy(t);
|
|
|
|
continuous_sequences[cf.name]=SContinuousSequence(
|
|
watoi64(extra_params[L"sequence_id"]), watoi64(extra_params[L"sequence_next"]));
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!dir_metadata.empty())
|
|
{
|
|
dir_metadata.pop();
|
|
}
|
|
|
|
--depth;
|
|
if(depth==0)
|
|
{
|
|
std::wstring t=curr_path;
|
|
t.erase(0,1);
|
|
if(t==L"urbackup_backup_scripts")
|
|
{
|
|
script_dir=false;
|
|
}
|
|
else
|
|
{
|
|
ServerLogger::Log(clientid, L"Stoping shadowcopy \""+t+L"\".", LL_DEBUG);
|
|
server_download->addToQueueStopShadowcopy(t);
|
|
}
|
|
}
|
|
curr_path=ExtractFilePath(curr_path, L"/");
|
|
curr_os_path=ExtractFilePath(curr_os_path, L"/");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
FileMetadata parent_metadata = dir_metadata.empty()?FileMetadata():dir_metadata.top();
|
|
|
|
bool file_ok=false;
|
|
std::map<std::wstring, std::wstring>::iterator hash_it=( (local_hash.get()==NULL)?extra_params.end():extra_params.find(L"sha512") );
|
|
if( hash_it!=extra_params.end())
|
|
{
|
|
if(link_file(cf.name, osspecific_name, curr_path, curr_os_path, base64_decode_dash(wnarrow(hash_it->second)), cf.size,
|
|
true, metadata, parent_metadata))
|
|
{
|
|
file_ok=true;
|
|
linked_bytes+=cf.size;
|
|
if(line>max_ok_id)
|
|
{
|
|
max_ok_id=line;
|
|
}
|
|
}
|
|
}
|
|
if(!file_ok)
|
|
{
|
|
server_download->addToQueueFull(line, cf.name, osspecific_name, curr_path, curr_os_path, queue_downloads?cf.size:-1,
|
|
metadata, parent_metadata, script_dir);
|
|
}
|
|
}
|
|
|
|
++line;
|
|
}
|
|
}
|
|
|
|
if(read<4096)
|
|
break;
|
|
}
|
|
|
|
server_download->queueStop(false);
|
|
|
|
ServerLogger::Log(clientid, L"Waiting for file transfers...", LL_INFO);
|
|
|
|
while(!Server->getThreadPool()->waitFor(server_download_ticket, 1000))
|
|
{
|
|
if(files_size==0)
|
|
{
|
|
status.pcdone=100;
|
|
}
|
|
else
|
|
{
|
|
status.pcdone=(std::min)(100,(int)(((float)fc.getReceivedDataBytes())/((float)files_size/100.f)+0.5f));
|
|
}
|
|
status.hashqueuesize=(_u32)hashpipe->getNumElements();
|
|
status.prepare_hashqueuesize=(_u32)hashpipe_prepare->getNumElements();
|
|
ServerStatus::setServerStatus(status, true);
|
|
|
|
int64 ctime = Server->getTimeMS();
|
|
if(ctime-last_eta_update>eta_update_intervall)
|
|
{
|
|
calculateEtaFileBackup(last_eta_update, ctime, fc, NULL, linked_bytes, last_eta_received_bytes, eta_estimated_speed, files_size);
|
|
}
|
|
}
|
|
|
|
if(server_download->isOffline() && !is_offline)
|
|
{
|
|
ServerLogger::Log(clientid, L"Client "+clientname+L" went offline.", LL_ERROR);
|
|
r_done=true;
|
|
}
|
|
|
|
size_t max_line = line;
|
|
|
|
if(r_done==false && c_has_error==false)
|
|
{
|
|
sendBackupOkay(true);
|
|
}
|
|
else
|
|
{
|
|
sendBackupOkay(false);
|
|
}
|
|
|
|
running_updater->stop();
|
|
backup_dao->updateFileBackupRunning(backupid);
|
|
|
|
ServerLogger::Log(clientid, L"Writing new file list...", LL_INFO);
|
|
|
|
tmp->Seek(0);
|
|
line = 0;
|
|
list_parser.reset();
|
|
while( (read=tmp->Read(buffer, 4096))>0 )
|
|
{
|
|
for(size_t i=0;i<read;++i)
|
|
{
|
|
bool b=list_parser.nextEntry(buffer[i], cf, NULL);
|
|
if(b)
|
|
{
|
|
if(cf.isdir && line<max_line)
|
|
{
|
|
writeFileItem(clientlist, cf);
|
|
}
|
|
else if(!cf.isdir &&
|
|
line <= (std::max)(server_download->getMaxOkId(), max_ok_id) &&
|
|
server_download->isDownloadOk(line) )
|
|
{
|
|
if(server_download->isDownloadPartial(line))
|
|
{
|
|
cf.last_modified *= Server->getRandomNumber();
|
|
}
|
|
writeFileItem(clientlist, cf);
|
|
}
|
|
++line;
|
|
}
|
|
}
|
|
}
|
|
|
|
Server->destroy(clientlist);
|
|
|
|
ServerLogger::Log(clientid, L"Waiting for file hashing and copying threads...", LL_INFO);
|
|
|
|
waitForFileThreads();
|
|
|
|
bool verification_ok = true;
|
|
if(server_settings->getSettings()->end_to_end_file_backup_verification
|
|
|| (client_main->isOnInternetConnection()
|
|
&& server_settings->getSettings()->verify_using_client_hashes
|
|
&& server_settings->getSettings()->internet_calculate_filehashes_on_client) )
|
|
{
|
|
if(!verify_file_backup(tmp))
|
|
{
|
|
ServerLogger::Log(clientid, "Backup verification failed", LL_ERROR);
|
|
c_has_error=true;
|
|
verification_ok = false;
|
|
}
|
|
else
|
|
{
|
|
ServerLogger::Log(clientid, "Backup verification ok", LL_INFO);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if( bsh->hasError() || bsh_prepare->hasError() )
|
|
{
|
|
disk_error=true;
|
|
}
|
|
else if(verification_ok)
|
|
{
|
|
db->BeginTransaction();
|
|
if(!os_rename_file(widen(clientlistName(group, true)), widen(clientlistName(group, false))) )
|
|
{
|
|
ServerLogger::Log(clientid, "Renaming new client file list to destination failed", LL_ERROR);
|
|
}
|
|
backup_dao->setFileBackupDone(backupid);
|
|
db->EndTransaction();
|
|
}
|
|
|
|
if( r_done==false && c_has_error==false && disk_error==false
|
|
&& (group==c_group_default || group==c_group_continuous))
|
|
{
|
|
std::wstring backupfolder=server_settings->getSettings()->backupfolder;
|
|
|
|
std::wstring name=L"current";
|
|
if(group==c_group_continuous)
|
|
{
|
|
name=L"continuous";
|
|
}
|
|
|
|
std::wstring currdir=backupfolder+os_file_sep()+clientname+os_file_sep()+name;
|
|
os_remove_symlink_dir(os_file_prefix(currdir));
|
|
os_link_symbolic(os_file_prefix(backuppath), os_file_prefix(currdir));
|
|
|
|
if(group==c_group_default)
|
|
{
|
|
currdir=backupfolder+os_file_sep()+L"clients";
|
|
if(!os_create_dir(os_file_prefix(currdir)) && !os_directory_exists(os_file_prefix(currdir)))
|
|
{
|
|
Server->Log("Error creating \"clients\" dir for symbolic links", LL_ERROR);
|
|
}
|
|
currdir+=os_file_sep()+clientname;
|
|
os_remove_symlink_dir(os_file_prefix(currdir));
|
|
os_link_symbolic(os_file_prefix(backuppath), os_file_prefix(currdir));
|
|
|
|
if(server_settings->getSettings()->create_linked_user_views)
|
|
{
|
|
ServerLogger::Log(clientid, "Creating user views...", LL_INFO);
|
|
|
|
createUserViews(tmp);
|
|
}
|
|
|
|
saveUsersOnClient();
|
|
}
|
|
}
|
|
|
|
{
|
|
std::wstring tmp_fn=tmp->getFilenameW();
|
|
Server->destroy(tmp);
|
|
Server->deleteFile(os_file_prefix(tmp_fn));
|
|
}
|
|
|
|
_i64 transferred_bytes=fc.getTransferredBytes();
|
|
int64 passed_time=Server->getTimeMS()-full_backup_starttime;
|
|
if(passed_time==0) passed_time=1;
|
|
|
|
ServerLogger::Log(clientid, "Transferred "+PrettyPrintBytes(transferred_bytes)+" - Average speed: "+PrettyPrintSpeed((size_t)((transferred_bytes*1000)/(passed_time)) ), LL_INFO );
|
|
|
|
ClientMain::run_script(L"urbackup" + os_file_sep() + L"post_full_filebackup", L"\""+ backuppath + L"\"", clientid);
|
|
|
|
if(c_has_error)
|
|
return false;
|
|
|
|
return !r_done;
|
|
}
|
|
|