mirrors/urbackup_backend
2
0
Fork 0
mirror of https://github.com/uroni/urbackup_backend.git synced 2025-10-26 11:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity
dev
urbackup_backend/urbackupserver/server_hash.cpp

1976 lines
51 KiB
C++
Raw Permalink Blame History

/*************************************************************************
* UrBackup - Client/Server backup system
* Copyright (C) 2011-2016 Martin Raiber
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/
#include "server_hash.h"
#include "server_prepare_hash.h"
#include "server_settings.h"
#include "server.h"
#include "../urbackupcommon/fileclient/FileClientChunked.h"
#include "../Interface/Server.h"
#include "../Interface/File.h"
#include "../Interface/Mutex.h"
#include "../common/data.h"
#include "database.h"
#include "../urbackupcommon/sha2/sha2.h"
#include "../stringtools.h"
#include "server_log.h"
#include "server_cleanup.h"
#include "create_files_index.h"
#include <algorithm>
#include <memory.h>
#include "../urbackupcommon/file_metadata.h"
#include "FileBackup.h"
#include <assert.h>
#ifdef _WIN32
#include <Windows.h>
#endif
const size_t freespace_mod=50*1024*1024; //50 MB
const size_t BUFFER_SIZE=64*1024; //64KB
IMutex * delete_mutex=NULL;
void init_mutex1(void)
{
delete_mutex=Server->createMutex();
}
void destroy_mutex1(void)
{
Server->destroy(delete_mutex);
}
BackupServerHash::BackupServerHash(IPipe *pPipe, int pClientid, bool use_snapshots, bool use_reflink, bool use_tmpfiles, logid_t logid,
bool snapshot_file_inplace, MaxFileId& max_file_id)
: use_snapshots(use_snapshots), use_reflink(use_reflink), use_tmpfiles(use_tmpfiles), filesdao(NULL), old_backupfolders_loaded(false),
logid(logid), snapshot_file_inplace(snapshot_file_inplace), max_file_id(max_file_id)
{
pipe=pPipe;
clientid=pClientid;
link_logcnt=0;
space_logcnt=0;
working=false;
has_error=false;
chunk_patcher.setCallback(this);
fileindex=NULL;
if(use_reflink)
ServerLogger::Log(logid, "Reflink copying is enabled", LL_DEBUG);
}
BackupServerHash::~BackupServerHash(void)
{
delete fileindex;
}
void BackupServerHash::setupDatabase(void)
{
db=Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER_FILES);
filesdao = new ServerFilesDao(db);
fileindex=create_lmdb_files_index();
}
void BackupServerHash::deinitDatabase(void)
{
db->freeMemory();
delete fileindex;
fileindex=NULL;
delete filesdao;
filesdao =NULL;
}
void BackupServerHash::operator()(void)
{
setupDatabase();
while(true)
{
working=false;
std::string data;
size_t rc=pipe->Read(&data, static_cast<int>(60000) );
if(rc==0)
{
link_logcnt=0;
space_logcnt=0;
continue;
}
working=true;
if(data=="exit")
{
pipe->Write("exit");
deinitDatabase();
Server->Log("server_hash Thread finished - normal");
Server->destroyDatabases(Server->getThreadID());
delete this;
return;
}
else if(data=="flush")
{
continue;
}
if(rc>0)
{
CRData rd(&data);
int iaction;
rd.getInt(&iaction);
EAction action=static_cast<EAction>(iaction);
if(action==EAction_LinkOrCopy)
{
int64 fileid;
rd.getVarInt(&fileid);
std::string temp_fn;
rd.getStr(&temp_fn);
int backupid;
rd.getInt(&backupid);
int incremental;
rd.getInt(&incremental);
char with_hashes;
rd.getChar(&with_hashes);
std::string tfn;
rd.getStr(&tfn);
std::string hashpath;
rd.getStr(&hashpath);
std::string sha2;
if(!rd.getStr(&sha2))
ServerLogger::Log(logid, "Reading hash from pipe failed", LL_ERROR);
if(sha2.size()!=SHA_DEF_DIGEST_SIZE)
ServerLogger::Log(logid, "SHA length of file hash of \""+tfn+"\" wrong.", LL_ERROR);
std::string hashoutput_fn;
rd.getStr(&hashoutput_fn);
std::string old_file_fn;
rd.getStr(&old_file_fn);
int64 t_filesize;
rd.getInt64(&t_filesize);
std::string sparse_extents_fn;
rd.getStr(&sparse_extents_fn);
FileMetadata metadata;
metadata.read(rd);
metadata.set_shahash(sha2);
IFile *tf=Server->openFile(os_file_prefix((temp_fn)), MODE_READ_SEQUENTIAL);
if(tf==NULL)
{
ServerLogger::Log(logid, "Error opening file \""+temp_fn+"\" from pipe for reading ec="+convert(os_last_error()), LL_ERROR);
has_error=true;
}
else
{
std::unique_ptr<ExtentIterator> extent_iterator;
if (!sparse_extents_fn.empty())
{
IFile* sparse_extents_f = Server->openFile(sparse_extents_fn, MODE_READ);
if (sparse_extents_f != NULL)
{
extent_iterator.reset(new ExtentIterator(sparse_extents_f));
}
}
addFile(backupid, incremental, tf, tfn, hashpath, sha2,
old_file_fn, hashoutput_fn, t_filesize, metadata, with_hashes!=0, extent_iterator.get(), fileid);
}
if(!hashoutput_fn.empty())
{
Server->deleteFile(hashoutput_fn);
}
max_file_id.setMaxDownloaded(fileid);
}
else if(action==EAction_Copy)
{
int64 fileid;
bool b = rd.getVarInt(&fileid);
assert(b);
std::string source;
b = rd.getStr(&source);
assert(b);
std::string dest;
b = rd.getStr(&dest);
assert(b);
std::string hash_src;
b = rd.getStr(&hash_src);
assert(b);
std::string hash_dest;
b = rd.getStr(&hash_dest);
assert(b);
FileMetadata metadata;
metadata.read(rd);
FileMetadata src_metadata;
if(read_metadata(os_file_prefix(hash_src),
src_metadata))
{
metadata.set_shahash(src_metadata.shahash);
}
std::unique_ptr<IFile> tf(Server->openFile(os_file_prefix(source), MODE_READ_SEQUENTIAL));
if(!tf.get())
{
ServerLogger::Log(logid, "Error opening file \""+source+"\" from pipe for reading ec="+convert(os_last_error()), LL_ERROR);
has_error=true;
}
else
{
if(!copyFile(tf.get(), dest, NULL))
{
ServerLogger::Log(logid, "Error while copying file \""+source+"\" to \""+dest+"\"", LL_ERROR);
has_error=true;
}
if(!hash_src.empty())
{
std::unique_ptr<IFile> hashf(Server->openFile(os_file_prefix(hash_src), MODE_READ_SEQUENTIAL));
if(hashf.get())
{
copyFile(hashf.get(), hash_dest, NULL);
}
}
}
}
}
}
}
void BackupServerHash::addFileSQL(int backupid, int clientid, int incremental, const std::string &fp, const std::string &hash_path, const std::string &shahash, _i64 filesize, _i64 rsize, int64 prev_entry, int64 prev_entry_clientid, int64 next_entry, bool update_fileindex)
{
addFileSQL(*filesdao, *fileindex, backupid, clientid, incremental, fp, hash_path, shahash, filesize, rsize, prev_entry, prev_entry_clientid, next_entry, update_fileindex);
}
void BackupServerHash::addFileSQL(ServerFilesDao& filesdao, FileIndex& fileindex, int backupid, const int clientid, int incremental, const std::string &fp,
const std::string &hash_path, const std::string &shahash, _i64 filesize, _i64 rsize, int64 prev_entry, int64 prev_entry_clientid, int64 next_entry, bool update_fileindex)
{
if (filesize < link_file_min_size)
{
assert(prev_entry_clientid == 0);
assert(prev_entry == 0);
assert(next_entry == 0);
filesdao.addIncomingFile(filesize, clientid, backupid, std::string(), ServerFilesDao::c_direction_incoming, incremental);
filesdao.addFileEntryExternal(backupid, fp, hash_path, shahash, filesize, rsize, clientid, incremental, next_entry, prev_entry, 0);
return;
}
bool new_for_client=false;
if(prev_entry_clientid!=clientid || prev_entry==0)
{
new_for_client=true;
//new file for this client (predicted)
prev_entry=0;
next_entry=0;
std::string clients;
if(prev_entry_clientid!=0)
{
//Other clients have this file
std::map<int, int64> all_clients = fileindex.get_all_clients_with_cache(FileIndex::SIndexKey(shahash.c_str(), filesize), true);
for(std::map<int, int64>::iterator it=all_clients.begin();it!=all_clients.end();++it)
{
if(it->second!=0)
{
if(it->first==clientid)
{
//client actually has this file, but it e.g. failed to link
prev_entry=it->second;
}
if(!clients.empty())
{
clients+=",";
}
clients+=convert(it->first);
}
}
}
if(prev_entry==0)
{
filesdao.addIncomingFile(filesize, clientid, backupid, clients, ServerFilesDao::c_direction_incoming, incremental);
}
else
{
ServerFilesDao::SFindFileEntry fentry = filesdao.getFileEntry(prev_entry);
if(fentry.exists)
{
next_entry = fentry.next_entry;
if(fentry.pointed_to!=0)
{
update_fileindex = true;
}
}
else
{
prev_entry=0;
}
}
}
if(update_fileindex)
{
//if prev_entry==0 the file is new for this client
//and pointed_to does not need to be updated
if(prev_entry!=0)
{
ServerFilesDao::CondInt64 fentry = filesdao.getPointedTo(prev_entry);
if(fentry.exists && fentry.value!=0)
{
filesdao.setPointedTo(0, prev_entry);
}
else
{
int64 client_entryid = fileindex.get_with_cache_exact(FileIndex::SIndexKey(shahash.c_str(), filesize, clientid));
if(client_entryid!=0)
{
filesdao.setPointedTo(0, client_entryid);
}
}
}
}
int64 entryid = filesdao.addFileEntryExternal(backupid, fp, hash_path, shahash, filesize, rsize, clientid, incremental, next_entry, prev_entry, (new_for_client || update_fileindex)?1:0);
if(new_for_client || update_fileindex)
{
FILEENTRY_DEBUG(Server->Log("New fileindex entry for \"" + fp + "\""
" id=" + convert(entryid)
+" hash="+base64_encode(reinterpret_cast<const unsigned char*>(shahash.c_str()), bytes_in_index), LL_DEBUG));
fileindex.put_delayed(FileIndex::SIndexKey(shahash.c_str(), filesize, clientid), entryid);
}
}
void BackupServerHash::deleteFileSQL(ServerFilesDao& filesdao, FileIndex& fileindex, int64 id)
{
ServerFilesDao::SFindFileEntry entry = filesdao.getFileEntry(id);
if(entry.exists)
{
deleteFileSQL(filesdao, fileindex, reinterpret_cast<const char*>(entry.shahash.c_str()),
entry.filesize, entry.rsize, entry.clientid, entry.backupid, entry.incremental,
id, entry.prev_entry, entry.next_entry, entry.pointed_to, true, true, true, false, NULL);
}
}
void BackupServerHash::deleteFileSQL(ServerFilesDao& filesdao, FileIndex& fileindex, const char* pHash, _i64 filesize, _i64 rsize, const int clientid, int backupid, int incremental, int64 id, int64 prev_id, int64 next_id, int pointed_to,
bool use_transaction, bool del_entry, bool detach_dbs, bool with_backupstat, SInMemCorrection* correction)
{
if(use_transaction)
{
filesdao.BeginWriteTransaction();
}
if(prev_id==0 && next_id==0)
{
if (filesize < link_file_min_size)
{
if (pointed_to != 0)
{
FILEENTRY_DEBUG(Server->Log("Small file entry with id " + convert(id) + " with filesize=" + convert(filesize)
+ " hash=" + base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " has pointed_to!=0 but should be zero. The file entry index may be damaged.", LL_WARNING));
}
filesdao.addIncomingFile(filesize, clientid, backupid, convert(clientid),
with_backupstat ? ServerFilesDao::c_direction_outgoing : ServerFilesDao::c_direction_outgoing_nobackupstat,
incremental);
if (del_entry)
{
filesdao.delFileEntry(id);
}
if (use_transaction)
{
filesdao.endTransaction();
}
return;
}
//client does not have this file anymore
std::map<int, int64> all_clients = fileindex.get_all_clients_with_cache(FileIndex::SIndexKey(pHash, filesize), true);
int64 target_entryid = 0;
std::string clients;
if(!all_clients.empty())
{
for(std::map<int, int64>::iterator it=all_clients.begin();it!=all_clients.end();++it)
{
if(it->second!=0)
{
if(!clients.empty())
{
clients+=",";
}
clients+=convert(it->first);
if (it->first == clientid)
{
target_entryid = it->second;
}
}
}
}
else
{
FILEENTRY_DEBUG(Server->Log("File entry with id "+convert(id)+" with filesize="+convert(filesize)
+ " hash="+base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " not found in entry index while deleting, but should be there. The file entry index may be damaged.", LL_WARNING));
clients+=convert(clientid);
}
if (target_entryid == 0)
{
FILEENTRY_DEBUG(Server->Log("File entry with id " + convert(id) + " with filesize=" + convert(filesize)
+ " hash=" + base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " not found for clientid "+convert(clientid)+" in file entry index while deleting, but should be there. The file entry index may be damaged.", LL_WARNING));
if (!clients.empty())
{
clients += ",";
}
clients += convert(clientid);
}
else if (target_entryid != id)
{
FILEENTRY_DEBUG(Server->Log("File entry with id " + convert(id) + " with filesize=" + convert(filesize) +
" hash=" + base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index) + " is the last file entry for this file and to be deleted. "
"However, the file entry index points to entry id "+convert(target_entryid)+" which differs. "
"The file entry index may be damaged or this was a small patch and the files are backed up with snapshots. Not deleting entry from file entry index", LL_WARNING));
}
if (!pointed_to)
{
FILEENTRY_DEBUG(Server->Log("File entry with id " + convert(id) + " with filesize=" + convert(filesize)
+ " hash=" + base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index) + " is the last file entry for this file and to be deleted. "
"However, pointed_to is zero, so it won't be deleted from the file entry index. The file entry index may be damaged.", LL_WARNING));
}
filesdao.addIncomingFile(filesize, clientid, backupid, clients,
with_backupstat? ServerFilesDao::c_direction_outgoing : ServerFilesDao::c_direction_outgoing_nobackupstat,
incremental);
if( pointed_to
&& !all_clients.empty()
&& (target_entryid==0 || target_entryid==id) )
{
FILEENTRY_DEBUG(Server->Log("Delete file index entry id=" + convert(id)+ " filesize="+convert(filesize)+" hash="
+ base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index), LL_DEBUG));
fileindex.del_delayed(FileIndex::SIndexKey(pHash, filesize, clientid));
}
}
else if(pointed_to)
{
if(next_id!=0
&& next_id!=id)
{
std::string str_correction;
if (correction != NULL
&& correction->needs_correction(next_id))
{
correction->pointed_to[next_id] = 1;
str_correction = " (with correction)";
}
else
{
filesdao.setPointedTo(1, next_id);
}
fileindex.put_delayed(FileIndex::SIndexKey(pHash, filesize, clientid), next_id);
FILEENTRY_DEBUG(Server->Log("Changed file index entry filesize="+convert(filesize)+" hash="
+ base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " from " + convert(id) + " to " + convert(next_id) + " (next)"+str_correction, LL_DEBUG));
}
else if(prev_id!=id)
{
std::string str_correction;
if (correction != NULL
&& correction->needs_correction(prev_id))
{
correction->pointed_to[prev_id] = 1;
str_correction = " (with correction)";
}
else
{
filesdao.setPointedTo(1, prev_id);
}
fileindex.put_delayed(FileIndex::SIndexKey(pHash, filesize, clientid), prev_id);
FILEENTRY_DEBUG(Server->Log("Changed file index entry filesize="+convert(filesize)+" hash = "
+ base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " from " + convert(id) + " to " + convert(prev_id) + " (prev)"+str_correction, LL_DEBUG));
}
else
{
FILEENTRY_DEBUG(Server->Log("Could not change file entry index with pointed_to=1 filesize=" + convert(filesize) + " hash = "
+ base64_encode(reinterpret_cast<const unsigned char*>(pHash), bytes_in_index)
+ " from " + convert(id) + " to " + convert(prev_id) + " (prev) or "+convert(next_id)+" (next)", LL_WARNING));
}
}
if(next_id!=0)
{
if (correction!=NULL
&& correction->needs_correction(next_id))
{
correction->prev_entries[next_id] = prev_id;
}
else
{
filesdao.setPrevEntry(prev_id, next_id);
}
}
if(prev_id!=0)
{
if (correction != NULL
&& correction->needs_correction(prev_id))
{
correction->next_entries[prev_id] = next_id;
}
else
{
filesdao.setNextEntry(next_id, prev_id);
}
}
if(del_entry)
{
filesdao.delFileEntry(id);
}
if(use_transaction)
{
filesdao.endTransaction();
}
}
bool BackupServerHash::findFileAndLink(const std::string &tfn, IFile *tf, std::string hash_fn, const std::string &sha2,
_i64 t_filesize, const std::string &hashoutput_fn, bool copy_from_hardlink_if_failed,
bool &tries_once, std::string &ff_last, bool &hardlink_limit, bool &copied_file, int64& entryid, int& entryclientid
, int64& rsize, int64& next_entry, FileMetadata& metadata, bool detach_dbs, ExtentIterator* extent_iterator)
{
hardlink_limit=false;
copied_file=false;
tries_once=false;
bool first_logmsg=true;
bool copy=true;
SFindState find_state;
ServerFilesDao::SFindFileEntry existing_file = findFileHash(sha2, t_filesize, clientid, find_state);
while(existing_file.exists)
{
ff_last=existing_file.fullpath;
tries_once=true;
bool too_many_hardlinks;
bool b = false;
if (use_snapshots
&& snapshot_file_inplace)
{
if (tf!=NULL
&& tf->Size() <= 8
&& os_get_file_type(os_file_prefix(tfn)) != 0
&& os_file_truncate(os_file_prefix(tfn), t_filesize) )
{
//zero patch
b = true;
}
else
{
Server->deleteFile(os_file_prefix(tfn));
}
}
if (!b)
{
b = FileBackup::create_hardlink(os_file_prefix(tfn), os_file_prefix(existing_file.fullpath), use_snapshots || (use_reflink && !BackupServer::canHardlink()), &too_many_hardlinks, &copy);
}
if(!b)
{
if(too_many_hardlinks)
{
ServerLogger::Log(logid, "HT: Hardlinking failed (Maximum hardlink count reached) Source=\""+existing_file.fullpath+"\" Destination=\""+tfn+"\"", LL_DEBUG);
hardlink_limit = true;
break;
}
else
{
hardlink_limit = false;
std::string errmsg;
int64 errcode = os_last_error(errmsg);
IFile *ctf=Server->openFile(os_file_prefix(existing_file.fullpath), MODE_READ);
if(ctf==NULL)
{
if(correctPath(existing_file.fullpath, existing_file.hashpath))
{
ServerLogger::Log(logid, "HT: Using new backupfolder for: \""+existing_file.fullpath+"\"", LL_DEBUG);
continue;
}
if(first_logmsg)
{
ServerLogger::Log(logid, "HT: Hardlinking failed (File doesn't exist): \""+existing_file.fullpath+"\"", LL_DEBUG);
}
first_logmsg=false;
deleteFileSQL(*filesdao, *fileindex, sha2.c_str(), t_filesize, existing_file.rsize, existing_file.clientid, existing_file.backupid, existing_file.incremental,
existing_file.id, existing_file.prev_entry, existing_file.next_entry, existing_file.pointed_to, true, true, detach_dbs, false, NULL);
existing_file = findFileHash(sha2, t_filesize, clientid, find_state);
}
else
{
ServerLogger::Log(logid, "HT: Hardlinking failed (unkown error) Source=\""+existing_file.fullpath+"\" Destination=\""+tfn+"\" -- "+errmsg+" (code: "+convert(errcode)+")", LL_DEBUG);
if(copy_from_hardlink_if_failed)
{
ServerLogger::Log(logid, "HT: Copying from file \""+existing_file.fullpath+"\"", LL_DEBUG);
if(!copyFile(ctf, tfn, extent_iterator))
{
ServerLogger::Log(logid, "Error copying file to destination -3", LL_ERROR);
has_error=true;
}
else
{
copied_file=true;
entryid = existing_file.id;
entryclientid = existing_file.clientid;
next_entry=existing_file.next_entry;
}
assert(!hash_fn.empty());
if(!existing_file.hashpath.empty())
{
std::unique_ptr<IFile> ctf_hash(Server->openFile(os_file_prefix(existing_file.hashpath), MODE_READ));
bool write_metadata = true;
if(ctf_hash.get()!=NULL)
{
int64 hashfilesize = read_hashdata_size(ctf_hash.get());
assert(hashfilesize == -1 || hashfilesize == t_filesize);
if (hashfilesize != -1)
{
if (!copyFile(ctf_hash.get(), hash_fn, NULL))
{
ServerLogger::Log(logid, "Error copying hashfile to destination -3", LL_ERROR);
has_error = true;
write_metadata = false;
}
else
{
if (!os_file_truncate(os_file_prefix(hash_fn), get_hashdata_size(t_filesize)))
{
ServerLogger::Log(logid, "Error truncating hashdata file -2. " + os_last_error_str(), LL_ERROR);
}
}
}
}
else
{
Server->Log("Error opening hash source file \""+existing_file.hashpath+"\". " + os_last_error_str(), LL_ERROR);
}
if (write_metadata && !write_file_metadata(hash_fn, this, metadata, false))
{
ServerLogger::Log(logid, "Error writing file metadata -2", LL_ERROR);
has_error = true;
}
}
copy=false;
}
Server->destroy(ctf);
break;
}
}
}
else //successfully linked file
{
entryid = existing_file.id;
entryclientid = existing_file.clientid;
next_entry=existing_file.next_entry;
assert(!hash_fn.empty());
if(existing_file.rsize!=0 &&
existing_file.rsize!=existing_file.filesize)
{
rsize=existing_file.rsize;
}
else
{
rsize=0;
}
metadata.rsize=rsize;
bool write_metadata=true;
if(!hashoutput_fn.empty())
{
std::unique_ptr<IFile> src(Server->openFile(os_file_prefix(hashoutput_fn), MODE_READ));
if(src.get()!=NULL)
{
if(!copyFile(src.get(), hash_fn, NULL))
{
ServerLogger::Log(logid, "Error copying hashoutput to destination -1", LL_ERROR);
has_error=true;
hash_fn.clear();
write_metadata = false;
}
else
{
if(!os_file_truncate(os_file_prefix(hash_fn), get_hashdata_size(t_filesize)))
{
ServerLogger::Log(logid, "Error truncating hashdata file -1. "+os_last_error_str(), LL_ERROR);
}
}
}
else
{
ServerLogger::Log(logid, "HT: Error opening hashoutput", LL_ERROR);
has_error=true;
hash_fn.clear();
write_metadata = false;
}
}
else if(!existing_file.hashpath.empty())
{
std::unique_ptr<IFile> ctf(Server->openFile(os_file_prefix(existing_file.hashpath), MODE_READ));
if(ctf.get()!=NULL)
{
int64 hashfilesize = read_hashdata_size(ctf.get());
if(hashfilesize!=-1
&& hashfilesize == t_filesize)
{
if(!copyFile(ctf.get(), hash_fn, NULL))
{
ServerLogger::Log(logid, "Error copying hashfile to destination -2", LL_ERROR);
has_error=true;
hash_fn.clear();
write_metadata = false;
}
else
{
if(!os_file_truncate(os_file_prefix(hash_fn), get_hashdata_size(t_filesize)))
{
ServerLogger::Log(logid, "Error truncating hashdata file -2. " + os_last_error_str(), LL_ERROR);
}
}
}
else
{
if (hashfilesize != -1)
{
Server->Log("File size in meta-data file \"" + existing_file.hashpath + "\" does not match database. From database=" + convert(t_filesize) + " In meta-data=" + convert(hashfilesize), LL_WARNING);
}
}
}
}
if(write_metadata && !write_file_metadata(hash_fn, this, metadata, false))
{
ServerLogger::Log(logid, "Error writing file metadata -1", LL_ERROR);
has_error=true;
}
copy=false;
break;
}
}
return !copy;
}
void BackupServerHash::addFile(int backupid, int incremental, IFile *tf, const std::string &tfn,
std::string hash_fn, const std::string &sha2, const std::string &orig_fn, const std::string &hashoutput_fn, int64 t_filesize,
FileMetadata& metadata, bool with_hashes, ExtentIterator* extent_iterator, int64 fileid)
{
bool copy=true;
bool tries_once = false;
std::string ff_last;
bool hardlink_limit = false;
bool copied_file = false;
int64 entryid = 0;
int entryclientid = 0;
int64 next_entryid = 0;
int64 rsize = 0;
if(t_filesize>= link_file_min_size
&& (!snapshot_file_inplace || t_filesize<50*1024*1024 || tf->Size()>10*1024 || tf->Size()<=8)
&& findFileAndLink(tfn, tf, hash_fn, sha2, t_filesize,hashoutput_fn,
false, tries_once, ff_last, hardlink_limit, copied_file, entryid, entryclientid, rsize, next_entryid,
metadata, false, extent_iterator))
{
ServerLogger::Log(logid, "HT: Linked file: \""+tfn+"\" (id="+convert(fileid)+")", LL_DEBUG);
copy=false;
std::string temp_fn=tf->getFilename();
Server->destroy(tf);
tf=NULL;
Server->deleteFile(temp_fn);
addFileSQL(backupid, clientid, incremental, tfn, hash_fn, sha2, t_filesize, rsize, entryid, entryclientid, next_entryid, copied_file);
}
if(tries_once && copy && !hardlink_limit)
{
if(link_logcnt<5)
{
ServerLogger::Log(logid, "HT: Error creating hardlink from \""+ff_last+"\" to \""+tfn+"\"", LL_WARNING);
}
else if(link_logcnt==5)
{
ServerLogger::Log(logid, "HT: More hardlink errors. Skipping... ", LL_WARNING);
}
else
{
Server->Log("HT: Error creating hardlink from \""+ff_last+"\" to \""+tfn+"\"", LL_WARNING);
}
++link_logcnt;
}
if(copy)
{
ServerLogger::Log(logid, "HT: Copying file: \""+tfn+"\" (id=" + convert(fileid) + ")", LL_DEBUG);
int64 fs=tf->Size();
if(!use_reflink)
{
fs=t_filesize;
}
int64 available_space=0;
cow_filesize=0;
if(fs>0)
{
available_space=os_free_space(os_file_prefix(ExtractFilePath(tfn, os_file_sep())));
}
if(available_space==-1)
{
if(space_logcnt==0)
{
ServerLogger::Log(logid, "HT: Error getting free space for path \""+tfn+"\"", LL_ERROR);
++space_logcnt;
}
else
{
ServerLogger::Log(logid, "HT: Error getting free space for path \""+tfn+"\"", LL_ERROR);
}
std::string temp_fn=tf->getFilename();
Server->destroy(tf);
tf=NULL;
Server->deleteFile(temp_fn);
}
else
{
available_space-=freespace_mod;
bool free_ok=true;
if( fs>0 && available_space<=fs )
{
if (extent_iterator != NULL)
{
extent_iterator->reset();
IFsFile::SSparseExtent ext;
while ((ext = extent_iterator->nextExtent()).offset!=-1)
{
if(ext.size>0)
fs -= ext.size;
}
extent_iterator->reset();
}
if (fs > 0 && available_space <= fs)
{
if (space_logcnt == 0)
{
ServerLogger::Log(logid, "HT: No free space available deleting backups...", LL_WARNING);
}
else
{
ServerLogger::Log(logid, "HT: No free space available deleting backups...", LL_WARNING);
}
free_ok = freeSpace(fs, os_file_prefix(tfn));
}
}
if(!free_ok)
{
if(space_logcnt==0)
{
ServerLogger::Log(logid, "HT: FATAL: Error freeing space", LL_ERROR);
++space_logcnt;
}
else
{
ServerLogger::Log(logid, "HT: FATAL: Error freeing space", LL_ERROR);
}
has_error=true;
std::string temp_fn=tf->getFilename();
Server->destroy(tf);
tf=NULL;
Server->deleteFile(temp_fn);
}
else
{
bool r;
if(hashoutput_fn.empty())
{
if(!use_reflink || orig_fn.empty())
{
if(with_hashes)
{
if(use_tmpfiles)
{
r=copyFileWithHashoutput(tf, tfn, hash_fn, extent_iterator);
}
else
{
r=renameFileWithHashoutput(tf, tfn, hash_fn, extent_iterator);
tf=NULL;
}
}
else
{
if(use_tmpfiles)
{
r=copyFile(tf, tfn, extent_iterator);
}
else
{
r=renameFile(tf, tfn);
tf=NULL;
}
}
}
else
{
if(with_hashes)
{
r=replaceFileWithHashoutput(tf, tfn, hash_fn, orig_fn, extent_iterator);
}
else
{
r=replaceFile(tf, tfn, orig_fn, extent_iterator);
}
}
}
else
{
r=patchFile(tf, orig_fn, tfn, hashoutput_fn, hash_fn, t_filesize, extent_iterator);
}
if(!r)
{
has_error=true;
ServerLogger::Log(logid, "Storing file to \""+tfn+"\" failed", LL_ERROR);
}
if(tf!=NULL)
{
metadata.rsize=tf->Size();
std::string temp_fn=tf->getFilename();
Server->destroy(tf);
tf=NULL;
Server->deleteFile(os_file_prefix(temp_fn));
}
if(r)
{
if(cow_filesize>0)
{
metadata.rsize=cow_filesize;
}
if(!write_file_metadata(hash_fn, this, metadata, false))
{
ServerLogger::Log(logid, "Writing metadata to "+hash_fn+" failed", LL_ERROR);
has_error=true;
}
addFileSQL(backupid, clientid, incremental, tfn, hash_fn, sha2, t_filesize, cow_filesize>0?cow_filesize:t_filesize, 0, 0, 0, tries_once || hardlink_limit);
}
}
}
}
}
bool BackupServerHash::freeSpace(int64 fs, const std::string &fp)
{
IScopedLock lock(delete_mutex);
int64 available_space=os_free_space(ExtractFilePath(fp, os_file_sep()));
if(available_space==-1)
{
if(space_logcnt==0)
{
ServerLogger::Log(logid, "Error getting free space for path \""+fp+"\"", LL_ERROR);
++space_logcnt;
}
else
{
ServerLogger::Log(logid, "Error getting free space for path \""+fp+"\"", LL_ERROR);
}
return false;
}
else
{
available_space-=freespace_mod;
}
if(available_space>fs)
return true;
bool b = ServerCleanupThread::cleanupSpace(freespace_mod+fs);
return b;
}
ServerFilesDao::SFindFileEntry BackupServerHash::findFileHash(const std::string &pHash, _i64 filesize, int clientid, SFindState& state)
{
int64 entryid;
bool save_orig=false;
bool switch_to_all_clients=false;
bool switch_to_next_client=false;
if(state.state==0)
{
entryid = fileindex->get_with_cache_prefer_client(FileIndex::SIndexKey(pHash.c_str(), filesize, clientid));
state.state=1;
save_orig=true;
}
else if(state.state==1)
{
if(state.prev.next_entry!=0)
{
entryid=state.prev.next_entry;
}
else
{
if(state.orig_prev!=0)
{
entryid = state.orig_prev;
state.state=2;
}
else
{
switch_to_all_clients=true;
}
}
}
else if(state.state==2)
{
if(state.prev.prev_entry!=0)
{
entryid=state.prev.prev_entry;
}
else
{
switch_to_all_clients=true;
}
}
else if(state.state==3)
{
if(state.prev.next_entry==0)
{
if(state.orig_prev!=0)
{
entryid = state.orig_prev;
state.state=4;
}
else
{
switch_to_next_client=true;
}
}
else
{
entryid=state.prev.next_entry;
}
}
else if(state.state==4)
{
if(state.prev.prev_entry==0)
{
switch_to_next_client=true;
}
else
{
entryid=state.prev.prev_entry;
}
}
if(switch_to_all_clients)
{
state.state=3;
state.entryids = fileindex->get_all_clients_with_cache(FileIndex::SIndexKey(pHash.c_str(), filesize, 0), false);
state.client = state.entryids.begin();
if(state.client!=state.entryids.end())
{
entryid=state.client->second;
save_orig=true;
}
else
{
entryid=0;
}
}
if(switch_to_next_client)
{
++state.client;
if(state.client==state.entryids.end())
{
entryid=0;
}
else
{
entryid=state.client->second;
}
}
if(entryid==0)
{
ServerFilesDao::SFindFileEntry ret;
ret.exists=false;
return ret;
}
state.prev = filesdao->getFileEntry(entryid);
if(!state.prev.exists)
{
ServerLogger::Log(logid, "Entry from file entry index not found. File entry index probably out of sync. (id="+convert(entryid)+")", LL_DEBUG);
ServerFilesDao::SFindFileEntry ret;
ret.exists=false;
return ret;
}
if(memcmp(state.prev.shahash.data(), pHash.data(), pHash.size())!=0)
{
ServerLogger::Log(logid, "Hash of file entry differs from file entry index result. Something may be wrong with the file entry index or this is a hash collision. Ignoring existing file and downloading anew.", LL_DEBUG);
ServerLogger::Log(logid, "While searching for file with size "+convert(filesize)+" and clientid "+convert(clientid)+". Resulting file path is \""+state.prev.fullpath+"\". (id="+convert(entryid)+")", LL_DEBUG);
ServerFilesDao::SFindFileEntry ret;
ret.exists=false;
return ret;
}
assert(state.prev.filesize == filesize);
if(save_orig && state.prev.exists)
{
state.orig_prev=state.prev.prev_entry;
}
return state.prev;
}
IFsFile* BackupServerHash::openFileRetry(const std::string &dest, int mode, std::string& errstr)
{
IFsFile *dst=NULL;
int count_t=0;
while(dst==NULL)
{
dst=Server->openFile(os_file_prefix(dest), mode);
if(dst==NULL)
{
errstr = os_last_error_str();
ServerLogger::Log(logid, "Error opening file... \""+dest+"\" retrying... "+ errstr, LL_DEBUG);
Server->wait(500);
++count_t;
if(count_t>=10)
{
ServerLogger::Log(logid, "Error opening file... \""+dest+"\". " + errstr, LL_ERROR);
return NULL;
}
}
}
return dst;
}
bool BackupServerHash::copyFile(IFile *tf, const std::string &dest, ExtentIterator* extent_iterator)
{
ServerLogger::Log(logid, "HT: Copying file to \""+dest+"\"", LL_DEBUG);
std::string errstr;
std::unique_ptr<IFsFile> dst(openFileRetry(dest, MODE_WRITE, errstr));
if (dst.get() == NULL)
{
ServerLogger::Log(logid, "Error opening dest file \"" + dest + "\". " + errstr, LL_ERROR);
return false;
}
tf->Seek(0);
_u32 read;
char buf[BUFFER_SIZE];
int64 fpos = 0;
IFsFile::SSparseExtent curr_extent;
if (extent_iterator != NULL)
{
curr_extent = extent_iterator->nextExtent();
}
int64 sparse_max = -1;
do
{
while (curr_extent.offset != -1
&& fpos >= curr_extent.offset
&& fpos <= curr_extent.offset+ curr_extent.size)
{
fpos = curr_extent.offset + curr_extent.size;
sparse_max = fpos;
if (!tf->Seek(fpos))
{
ServerLogger::Log(logid, "Error seeking in source file for sparse extent \"" + tf->getFilename() + "\" -2. " + os_last_error_str(), LL_ERROR);
return false;
}
if (!punchHoleOrZero(dst.get(), curr_extent.offset, curr_extent.size))
{
ServerLogger::Log(logid, "Error adding sparse extent to \"" + dest + "\"", LL_ERROR);
return false;
}
if (!dst->Seek(fpos))
{
ServerLogger::Log(logid, "Error seeking in \"" + dest + "\" after adding sparse extent", LL_ERROR);
return false;
}
curr_extent = extent_iterator->nextExtent();
}
_u32 toread = BUFFER_SIZE;
if (curr_extent.offset != -1
&& fpos + toread > curr_extent.offset)
{
toread = (std::min)(toread, static_cast<_u32>(curr_extent.offset - fpos));
}
bool has_read_error = false;
read=tf->Read(buf, toread, &has_read_error);
if (has_read_error)
{
ServerLogger::Log(logid, "Error while reading from \"" + tf->getFilename() + "\" while copying to \""+dest+"\"", LL_ERROR);
return false;
}
bool b=writeRepeatFreeSpace(dst.get(), buf, read, this);
if(!b)
{
ServerLogger::Log(logid, "Error writing to file \""+dest+"\" -2. "+os_last_error_str(), LL_ERROR);
return false;
}
else
{
fpos += read;
}
}
while(read>0);
if (sparse_max!=-1
&& sparse_max > dst->Size())
{
sparse_max = (std::min)(tf->Size(), sparse_max);
if (!dst->Resize(sparse_max))
{
ServerLogger::Log(logid, "Error resizing file \"" + dest + "\" to max sparse size " + convert(sparse_max), LL_ERROR);
return false;
}
}
return true;
}
bool BackupServerHash::copyFileWithHashoutput(IFile *tf, const std::string &dest, const std::string hash_dest, ExtentIterator* extent_iterator)
{
ServerLogger::Log(logid, "HT: Copying file with hash output to \""+dest+"\"", LL_DEBUG);
std::string errstr;
IFsFile *dst=openFileRetry(dest, MODE_WRITE, errstr);
if(dst==NULL) return false;
ObjectScope dst_s(dst);
if(tf->Size()>0)
{
IFile *dst_hash=openFileRetry(hash_dest, MODE_RW_CREATE, errstr);
if(dst_hash==NULL)
{
return false;
}
ObjectScope dst_hash_s(dst_hash);
return build_chunk_hashs(tf, dst_hash, this, dst, false, NULL, NULL, false, NULL, extent_iterator);
}
return true;
}
bool BackupServerHash::isWorking(void)
{
return working;
}
bool BackupServerHash::hasError(void)
{
volatile bool r=has_error;
has_error=false;
return r;
}
bool BackupServerHash::handle_not_enough_space(const std::string &path)
{
int64 available_space=os_free_space(ExtractFilePath(path));
if(available_space==-1)
{
if(space_logcnt==0)
{
ServerLogger::Log(logid, "Error writing to file \""+path+"\". "+os_last_error_str(), LL_ERROR);
++space_logcnt;
}
else
{
ServerLogger::Log(logid, "Error writing to file \""+path+"\". "+os_last_error_str(), LL_ERROR);
}
return false;
}
else
{
if(available_space<=freespace_mod)
{
if(space_logcnt==0)
{
ServerLogger::Log(logid, "HT: No free space available deleting backups...", LL_WARNING);
}
else
{
ServerLogger::Log(logid, "HT: No free space available deleting backups...", LL_WARNING);
}
return freeSpace(0, path);
}
return true;
}
}
void BackupServerHash::next_chunk_patcher_bytes(const char *buf, size_t bsize, bool changed, bool* is_sparse)
{
if(!has_reflink || changed )
{
if (buf != NULL) //buf is NULL for sparse extents
{
if (!chunk_output_fn->Seek(chunk_patch_pos))
{
ServerLogger::Log(logid, "Error seeking to offset "+convert(chunk_patch_pos)+" in \"" + chunk_output_fn->getFilename() + "\" -3", LL_ERROR);
chunk_patcher_has_error = true;
}
bool b = writeRepeatFreeSpace(chunk_output_fn, buf, bsize, this);
if (!b)
{
ServerLogger::Log(logid, "Error writing to file \"" + chunk_output_fn->getFilename() + "\" -3. "+os_last_error_str(), LL_ERROR);
chunk_patcher_has_error = true;
}
}
else
{
#ifdef _WIN32
if (!enabled_sparse)
{
//Punch hole once to enable sparse file on Windows
chunk_output_fn->PunchHole(chunk_patch_pos, bsize);
enabled_sparse = true;
}
#endif
}
}
chunk_patch_pos+=bsize;
if(has_reflink && changed)
{
cow_filesize+=bsize;
}
}
void BackupServerHash::next_sparse_extent_bytes(const char * buf, size_t bsize)
{
}
int64 BackupServerHash::chunk_patcher_pos()
{
return chunk_patch_pos;
}
bool BackupServerHash::patchFile(IFile *patch, const std::string &source, const std::string &dest,
const std::string hash_output, const std::string hash_dest, _i64 tfilesize, ExtentIterator* extent_iterator)
{
_i64 dstfsize;
{
has_reflink=false;
bool copy = false;
if( use_reflink )
{
if( (!snapshot_file_inplace || (os_get_file_type(os_file_prefix(dest)) & EFileType_File)==0)
&& !FileBackup::create_hardlink(os_file_prefix(dest), os_file_prefix(source), true, NULL, &copy) )
{
ServerLogger::Log(logid, "Reflinking file \""+dest+"\" failed", LL_WARNING);
}
else
{
has_reflink=!copy;
}
}
ServerLogger::Log(logid, "HT: Patching file \""+dest+"\" with source \""+source+"\" patch size "+PrettyPrintBytes(patch->Size())+" reflink="+convert(has_reflink), LL_DEBUG);
std::string errstr;
chunk_output_fn=openFileRetry(dest, has_reflink?MODE_RW:MODE_WRITE, errstr);
if (chunk_output_fn == NULL)
{
ServerLogger::Log(logid, "Error opening chunk output file \"" + dest + "\". "+ errstr, LL_ERROR);
return false;
}
ObjectScope dst_s(chunk_output_fn);
IFile *f_source=openFileRetry(source, MODE_READ, errstr);
if (f_source == NULL)
{
ServerLogger::Log(logid, "Error opening patch source file \"" + source + "\". "+errstr, LL_ERROR);
return false;
}
ObjectScope f_source_s(f_source);
chunk_patch_pos=0;
enabled_sparse = false;
chunk_patcher_has_error = false;
chunk_patcher.setRequireUnchanged(!has_reflink);
bool b=chunk_patcher.ApplyPatch(f_source, patch, extent_iterator);
if (!b)
{
ServerLogger::Log(logid, "Error applying patch to \"" + dest + "\" with source \"" + source + "\"", LL_ERROR);
}
IFsFile::SSparseExtent sparse_extent;
if (extent_iterator != NULL)
{
extent_iterator->reset();
sparse_extent = extent_iterator->nextExtent();
}
dstfsize = chunk_output_fn->Size();
bool sparse_resize = false;
bool has_punch_error = false;
while (sparse_extent.offset != -1)
{
if (!punchHoleOrZero(chunk_output_fn, sparse_extent.offset, sparse_extent.size))
{
if (!has_punch_error)
{
ServerLogger::Log(logid, "Error punching hole into \"" + dest + "\"", LL_ERROR);
}
has_punch_error = true;
chunk_patcher_has_error = true;
}
if (sparse_extent.offset + sparse_extent.size > dstfsize)
{
sparse_resize = true;
dstfsize = sparse_extent.offset + sparse_extent.size;
}
sparse_extent = extent_iterator->nextExtent();
}
if (sparse_resize)
{
if (!chunk_output_fn->Resize(dstfsize))
{
ServerLogger::Log(logid, "Error resizing \"" + dest + "\" to " + convert(dstfsize), LL_ERROR);
return false;
}
}
if(chunk_patcher_has_error || !b)
{
return false;
}
}
assert(chunk_patcher.getFilesize() == tfilesize);
if( dstfsize > chunk_patcher.getFilesize() )
{
if (!os_file_truncate(dest, chunk_patcher.getFilesize()))
{
ServerLogger::Log(logid, "Error truncating \""+dest+"\" to "+convert(chunk_patcher.getFilesize())+". "+os_last_error_str(), LL_ERROR);
return false;
}
}
else
{
if (dstfsize != tfilesize)
{
ServerLogger::Log(logid, "dstfsize="+convert(dstfsize)+" tfilesize="+convert(tfilesize)+" dest=\""+ dest+"\" patch=\""+patch->getFilename()+"\" source=\""+ source+"\"", LL_ERROR);
}
assert(dstfsize==tfilesize);
}
IFile *f_hash_output=Server->openFile(os_file_prefix(hash_output), MODE_READ);
if(f_hash_output==NULL)
{
ServerLogger::Log(logid, "Error opening hashoutput file -1", LL_ERROR);
return false;
}
ObjectScope f_hash_output_s(f_hash_output);
if(!copyFile(f_hash_output, hash_dest, NULL))
{
ServerLogger::Log(logid, "Error copying hashoutput file to destination", LL_ERROR);
return false;
}
return true;
}
const size_t RP_COPY_BLOCKSIZE=1024;
bool BackupServerHash::replaceFile(IFile *tf, const std::string &dest, const std::string &orig_fn, ExtentIterator* extent_iterator)
{
bool copy = false;
if( (!snapshot_file_inplace || (os_get_file_type(os_file_prefix(dest)) & EFileType_File) == 0)
&& ! FileBackup::create_hardlink(os_file_prefix(dest), os_file_prefix(orig_fn), true, NULL, &copy) )
{
ServerLogger::Log(logid, "Reflinking file \""+dest+"\" failed -2", LL_ERROR);
return copyFile(tf, dest, extent_iterator);
}
ServerLogger::Log(logid, "HT: Copying with reflink data from \""+orig_fn+"\"", LL_DEBUG);
std::string errstr;
std::unique_ptr<IFile> dst(openFileRetry(dest, MODE_RW, errstr));
if (dst.get() == NULL)
{
ServerLogger::Log(logid, "Error opening destination file at \"" + dest + "\". " + errstr, LL_ERROR);
return false;
}
IFsFile::SSparseExtent curr_extent;
if (extent_iterator != NULL)
{
extent_iterator->reset();
curr_extent = extent_iterator->nextExtent();
}
tf->Seek(0);
_u32 read1;
_u32 read2;
char buf1[RP_COPY_BLOCKSIZE];
char buf2[RP_COPY_BLOCKSIZE];
_i64 dst_pos=0;
bool dst_eof=false;
do
{
while (curr_extent.offset != -1
&& curr_extent.offset >= dst_pos
&& curr_extent.offset + curr_extent.size > dst_pos)
{
if (!punchHoleOrZero(dst.get(), curr_extent.offset, curr_extent.size))
{
ServerLogger::Log(logid, "Error punching hole into \""+dest+"\"", LL_ERROR);
return false;
}
dst_pos = curr_extent.offset + curr_extent.size;
if (!tf->Seek(dst_pos) || !dst->Seek(dst_pos))
{
ServerLogger::Log(logid, "Error seeking after punching hole in replaceFile", LL_ERROR);
return false;
}
curr_extent = extent_iterator->nextExtent();
}
bool has_read_error = false;
read1=tf->Read(buf1, RP_COPY_BLOCKSIZE, &has_read_error);
if(!dst_eof)
{
read2=dst->Read(buf2, RP_COPY_BLOCKSIZE);
}
else
{
read2=0;
}
if (has_read_error)
{
ServerLogger::Log(logid, "Error reading from \"" + tf->getFilename() + "\" -1", LL_ERROR);
return false;
}
if(read2<read1)
dst_eof=true;
if(read1!=read2 || memcmp(buf1, buf2, read1)!=0)
{
dst->Seek(dst_pos);
cow_filesize+=read1;
bool b=writeRepeatFreeSpace(dst.get(), buf1, read1, this);
if(!b)
{
ServerLogger::Log(logid, "Error writing to file \""+dest+"\" -2. "+os_last_error_str(), LL_ERROR);
return false;
}
}
else if (copy)
{
cow_filesize += read1;
}
dst_pos+=read1;
}
while(read1>0);
_i64 dst_size=dst->Size();
dst.reset();
if( dst_size!=tf->Size() )
{
if( !os_file_truncate(os_file_prefix(dest), tf->Size()) )
{
ServerLogger::Log(logid, "Error truncating file \""+dest+"\" -2. "+os_last_error_str(), LL_ERROR);
return false;
}
}
return true;
}
bool BackupServerHash::replaceFileWithHashoutput(IFile *tf, const std::string &dest,
const std::string hash_dest, const std::string &orig_fn, ExtentIterator* extent_iterator)
{
bool copy = false;
if( (!snapshot_file_inplace || (os_get_file_type(os_file_prefix(dest)) & EFileType_File) == 0)
&& !FileBackup::create_hardlink(os_file_prefix(dest), os_file_prefix(orig_fn), true, NULL, &copy) )
{
ServerLogger::Log(logid, "Reflinking file \""+dest+"\" failed -3", LL_ERROR);
return copyFileWithHashoutput(tf, dest, hash_dest, extent_iterator);
}
ServerLogger::Log(logid, "HT: Copying with hashoutput with reflink data from \""+orig_fn+"\"", LL_DEBUG);
std::string errstr;
IFsFile *dst=openFileRetry(os_file_prefix(dest), MODE_RW, errstr);
if (dst == NULL)
{
ServerLogger::Log(logid, "Error opening dest file at \"" + dest + "\". "+ errstr, LL_ERROR);
return false;
}
ObjectScope dst_s(dst);
if(tf->Size()>0)
{
IFile *dst_hash=openFileRetry(os_file_prefix(hash_dest), MODE_WRITE, errstr);
if(dst_hash==NULL)
{
ServerLogger::Log(logid, "Error opening dest hash file at \"" + hash_dest + "\". "+ errstr, LL_ERROR);
return false;
}
ObjectScope dst_hash_s(dst_hash);
if (!build_chunk_hashs(tf, dst_hash, this, dst, true, &cow_filesize, NULL, false, NULL, extent_iterator))
{
ServerLogger::Log(logid, "Error copying file with chunk hashes to \""+dest+"\"", LL_ERROR);
return false;
}
_i64 dst_size=dst->Size();
if (copy)
{
cow_filesize = dst_size;
}
dst_s.clear();
if( dst_size!=tf->Size() )
{
if( !os_file_truncate(os_file_prefix(dest), tf->Size()) )
{
ServerLogger::Log(logid, "Error truncating file \""+dest+"\" -2. "+os_last_error_str(), LL_ERROR);
return false;
}
}
}
return true;
}
bool BackupServerHash::renameFileWithHashoutput(IFile *tf, const std::string &dest, const std::string hash_dest, ExtentIterator* extent_iterator)
{
ServerLogger::Log(logid, "HT: Renaming file to \""+dest+"\" with hash output", LL_DEBUG);
if(tf->Size()>0)
{
std::string errstr;
IFile *dst_hash=openFileRetry(hash_dest, MODE_RW_CREATE, errstr);
if(dst_hash==NULL)
{
return false;
}
ObjectScope dst_hash_s(dst_hash);
//TODO: Already build hashes during hashing stage
if (!build_chunk_hashs(tf, dst_hash, this, NULL, false, NULL, NULL, false, NULL, extent_iterator))
{
return false;
}
}
return renameFile(tf, dest, false);
}
bool BackupServerHash::renameFile(IFile *tf, const std::string &dest, bool log_info)
{
if(log_info)
{
ServerLogger::Log(logid, "HT: Renaming file to \""+dest+"\"", LL_DEBUG);
}
std::string tf_fn=tf->getFilename();
Server->destroy(tf);
if (!os_rename_file(os_file_prefix(tf_fn), os_file_prefix(dest)))
{
std::string err = os_last_error_str();
if (use_reflink &&
FileBackup::create_hardlink(os_file_prefix(dest), os_file_prefix(tf_fn), true, NULL, NULL))
{
Server->deleteFile(os_file_prefix(tf_fn));
}
else
{
ServerLogger::Log(logid, "Renaming file \""+ tf_fn+"\" to \"" + dest + "\" failed -4. "+err, LL_ERROR);
return false;
}
}
return true;
}
bool BackupServerHash::correctPath( std::string& ff, std::string& f_hashpath )
{
if(!old_backupfolders_loaded)
{
old_backupfolders_loaded=true;
ServerBackupDao backupdao(Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER));
old_backupfolders= backupdao.getOldBackupfolders();
}
if(backupfolder.empty())
{
ServerSettings settings(Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER));
backupfolder = settings.getSettings()->backupfolder;
}
if (correctClientName(backupfolder, ff, f_hashpath))
{
return true;
}
for(size_t i=0;i<old_backupfolders.size();++i)
{
size_t erase_size = old_backupfolders[i].size() + os_file_sep().size();
if(ff.size()>erase_size &&
next(ff, 0, old_backupfolders[i] + os_file_sep()))
{
std::string tmp_ff = backupfolder + os_file_sep() + ff.substr(erase_size);
std::string tmp_hashff;
if (f_hashpath.size() > erase_size)
{
tmp_hashff = backupfolder + os_file_sep() + f_hashpath.substr(erase_size);
}
IFile* f = Server->openFile(os_file_prefix(tmp_ff), MODE_READ);
if(f!=NULL)
{
Server->destroy(f);
f_hashpath = tmp_hashff;
ff = tmp_ff;
return true;
}
else if (correctClientName(backupfolder, tmp_ff, tmp_hashff))
{
return true;
}
}
}
return false;
}
bool BackupServerHash::correctClientName(const std::string& backupfolder, std::string & ff, std::string & f_hashpath)
{
if (ff.size() <= backupfolder.size()
|| f_hashpath.size() <= backupfolder.size())
return false;
size_t clientname_off = ff.find(os_file_sep(), backupfolder.size() + os_file_sep().size());
if (clientname_off == std::string::npos)
return false;
std::string clientname = ff.substr(backupfolder.size() + 1, clientname_off - backupfolder.size() - 1);
ServerBackupDao backupdao(Server->getDatabase(Server->getThreadID(), URBACKUPDB_SERVER));
std::map<std::string, std::vector<std::string> >::iterator it = client_moved_to.find(clientname);
if (it == client_moved_to.end())
{
client_moved_to[clientname] = backupdao.getClientMovedFrom(clientname);
it = client_moved_to.find(clientname);
}
if (it->second.empty())
return false;
for (size_t i = 0; i < it->second.size(); ++i)
{
std::string tmp_ff = backupfolder + os_file_sep() + it->second[i] + ff.substr(clientname_off);
IFile* f = Server->openFile(os_file_prefix(tmp_ff), MODE_READ);
if (f != NULL)
{
Server->destroy(f);
if (f_hashpath.size()>clientname_off)
{
f_hashpath = backupfolder + os_file_sep() + it->second[i] + f_hashpath.substr(clientname_off);
}
ff = tmp_ff;
return true;
}
}
return false;
}
bool BackupServerHash::punchHoleOrZero(IFile * tf, int64 offset, int64 size)
{
if (!tf->PunchHole(offset, size))
{
std::vector<char> zero_buf;
zero_buf.resize(32768);
if (tf->Seek(offset))
{
for (int64 written = 0; written < size;)
{
_u32 towrite = static_cast<_u32>((std::min)(size - written, static_cast<int64>(zero_buf.size())));
if (!writeRepeatFreeSpace(tf, zero_buf.data(), towrite, this))
{
ServerLogger::Log(logid, "Error zeroing data in \"" + chunk_output_fn->getFilename() + "\" after punching hole failed -2. " + os_last_error_str(), LL_ERROR);
return false;
}
written += towrite;
}
}
else
{
ServerLogger::Log(logid, "Error zeroing data in \"" + chunk_output_fn->getFilename() + "\" after punching hole failed -1. "+os_last_error_str(), LL_ERROR);
return false;
}
}
return true;
}
Reference in New Issue View Git Blame Copy Permalink