mirrors/urbackup_backend
2
0
Fork 0
mirror of https://github.com/uroni/urbackup_backend.git synced 2025-10-26 11:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity
bbb9dc66ff
urbackup_backend/fileservplugin/CClientThread.cpp

2883 lines
64 KiB
C++
Raw Blame History

/*************************************************************************
* UrBackup - Client/Server backup system
* Copyright (C) 2011-2016 Martin Raiber
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/
#include "../vld.h"
#include "../Interface/Server.h"
#include "../Interface/Pipe.h"
#include "../Interface/Mutex.h"
#include "../Interface/Condition.h"
#include "../Interface/File.h"
#include "CClientThread.h"
#include "CTCPFileServ.h"
#include "map_buffer.h"
#include "log.h"
#include "packet_ids.h"
#include "../stringtools.h"
#include "CriticalSection.h"
#include "FileServ.h"
#include "ChunkSendThread.h"
#include "../urbackupcommon/sha2/sha2.h"
#include "../common/adler32.h"
#include <memory>
#include <algorithm>
#include <limits.h>
#include <memory.h>
#include "FileServFactory.h"
#include "../urbackupcommon/os_functions.h"
#include "PipeSessions.h"
#ifndef _WIN32
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
#include <assert.h>
#if defined(__APPLE__) || defined(__FreeBSD__)
#include <sys/uio.h>
#define open64 open
#define off64_t off_t
#define lseek64 lseek
#define O_LARGEFILE 0
#define stat64 stat
#define fstat64 fstat
#if defined(__FreeBSD__)
#define sendfile64(a, b, c, d, e) sendfile(a, b, c, d, NULL, e, 0)
#else
#define sendfile64(a, b, c, d, e) sendfile(a, b, c, e, NULL, 0)
#endif
#endif
#include "FileMetadataPipe.h"
#ifdef __linux__
#ifndef SEEK_DATA
#define SEEK_DATA 3
#endif
#ifndef SEEK_HOLE
#define SEEK_HOLE 4
#endif
#endif
#if defined(__FreeBSD__)
//2017-05-12: Make illegal. Performance problems with ZFS on FreeBSD
#undef SEEK_HOLE
#define SEEK_HOLE 400
#endif
#define CLIENT_TIMEOUT 120
#define CHECK_BASE_PATH
#define SEND_TIMEOUT 300000
CClientThread::CClientThread(SOCKET pSocket, CTCPFileServ* pParent)
: extra_buffer(NULL), waiting_for_chunk(false),
backup_semantics(true), is_tunneled(false)
{
int_socket=pSocket;
stopped=false;
killable=false;
has_socket=true;
parent=pParent;
#ifdef _WIN32
bufmgr=new fileserv::CBufMgr(NBUFFERS, READSIZE);
#else
bufmgr=NULL;
#endif
hFile=INVALID_HANDLE_VALUE;
#ifdef _WIN32
int window_size = Server->getSendWindowSize();
if (window_size > 0)
{
int err = setsockopt(pSocket, SOL_SOCKET, SO_SNDBUF, (char *)&window_size, sizeof(window_size));
if (err == SOCKET_ERROR)
Log("Error: Can't modify SO_SNDBUF", LL_DEBUG);
}
window_size = Server->getRecvWindowSize();
if (window_size > 0)
{
int err = setsockopt(pSocket, SOL_SOCKET, SO_RCVBUF, (char *)&window_size, sizeof(window_size));
if (err == SOCKET_ERROR)
Log("Error: Can't modify SO_RCVBUF", LL_DEBUG);
}
#endif
close_the_socket=true;
errcount=0;
clientpipe=Server->PipeFromSocket(pSocket);
mutex=NULL;
cond=NULL;
state=CS_NONE;
chunk_send_thread_ticket=ILLEGAL_THREADPOOL_TICKET;
}
CClientThread::CClientThread(IPipe *pClientpipe, CTCPFileServ* pParent, std::vector<char>* extra_buffer)
: extra_buffer(extra_buffer), waiting_for_chunk(false), is_tunneled(true)
{
stopped=false;
killable=false;
has_socket=false;
parent=pParent;
bufmgr=NULL;
hFile=INVALID_HANDLE_VALUE;
close_the_socket=false;
errcount=0;
clientpipe=pClientpipe;
state=CS_NONE;
mutex=NULL;
cond=NULL;
chunk_send_thread_ticket=ILLEGAL_THREADPOOL_TICKET;
stack.setAddChecksum(true);
}
CClientThread::~CClientThread()
{
delete bufmgr;
if(mutex!=NULL)
{
Server->destroy(mutex);
Server->destroy(cond);
}
if(close_the_socket)
{
Server->destroy(clientpipe);
}
}
void CClientThread::operator()(void)
{
#ifdef BACKGROUND_PRIORITY
ScopedBackgroundPrio background_prio(false);
if(FileServFactory::backgroundBackupsEnabled())
{
#ifndef _DEBUG
background_prio.enable();
#endif
}
#endif
backup_semantics = FileServFactory::backupSemanticsEnabled();
while( RecvMessage() && !stopped )
{
}
ReleaseMemory();
if( hFile!=INVALID_HANDLE_VALUE )
{
CloseHandle( hFile );
hFile=INVALID_HANDLE_VALUE;
}
if(chunk_send_thread_ticket!=ILLEGAL_THREADPOOL_TICKET)
{
{
IScopedLock lock(mutex);
state=CS_NONE;
while(!next_chunks.empty())
{
if(next_chunks.front().update_file!=NULL)
{
if (next_chunks.front().pipe_file_user == NULL)
{
FileServ::decrShareActive(next_chunks.front().s_filename, next_chunks.front().share_active_gen);
Server->destroy(next_chunks.front().update_file);
}
delete next_chunks.front().pipe_file_user;
}
next_chunks.pop();
}
cond->notify_all();
}
Server->getThreadPool()->waitFor(chunk_send_thread_ticket);
}
killable=true;
}
bool CClientThread::RecvMessage()
{
_i32 rc;
if(extra_buffer==NULL || extra_buffer->empty())
{
rc=clientpipe->isReadable(60*1000)?1:0;
if(clientpipe->hasError())
rc=-1;
if( rc==0 )
{
Log("1 min Timeout deleting Buffers ("+convert((NBUFFERS*READSIZE)/1024 )+" KB) and waiting 1h more...", LL_DEBUG);
delete bufmgr;
bufmgr=NULL;
int n=0;
while(!stopped && rc==0 && n<60)
{
if (chunk_send_thread_ticket != ILLEGAL_THREADPOOL_TICKET
&& state == CS_BLOCKHASH)
{
IScopedLock lock(mutex);
if (next_chunks.empty()
&& waiting_for_chunk)
{
queueChunk(SChunk(ID_FREE_SERVER_FILE));
}
}
rc=clientpipe->isReadable(60*1000)?1:0;
if(clientpipe->hasError())
rc=-1;
++n;
}
}
}
else
{
rc=1;
}
if(rc<1)
{
Log("Select Error/Timeout in RecvMessage", LL_DEBUG);
return false;
}
else
{
if(extra_buffer==NULL || extra_buffer->empty())
{
rc=(_i32)clientpipe->Read(buffer, BUFFERSIZE, 60*1000);
if (rc > 0)
{
Server->Log("FileSRV received " + convert(rc));
}
}
else
{
size_t tocopy = (std::min)((size_t)BUFFERSIZE, extra_buffer->size());
memcpy(buffer, extra_buffer->data(), tocopy);
extra_buffer->erase(extra_buffer->begin(), extra_buffer->begin()+tocopy);
rc = static_cast<_i32>(tocopy);
}
if(rc<1)
{
Log("Recv Error in RecvMessage", LL_DEBUG);
return false;
}
else
{
Log("Received data...");
stack.AddData(buffer, rc);
}
size_t packetsize;
char* packet;
while( (packet=stack.getPacket(&packetsize)) != NULL )
{
Log("Received a Packet.", LL_DEBUG);
CRData data(packet, packetsize);
bool b=ProcessPacket( &data );
delete[] packet;
if( !b )
return false;
}
}
return true;
}
int CClientThread::SendInt(const char *buf, size_t bsize, bool flush)
{
if(bsize==0)
{
clientpipe->shutdown();
return 0;
}
else
{
return (int)(clientpipe->Write(buf, bsize, SEND_TIMEOUT, flush)?bsize:SOCKET_ERROR);
}
}
bool CClientThread::FlushInt()
{
return clientpipe->Flush(CLIENT_TIMEOUT * 1000);
}
bool CClientThread::ProcessPacket(CRData *data)
{
uchar id;
if( data->getUChar(&id)==true )
{
switch(id)
{
case ID_GET_FILE:
case ID_GET_FILE_RESUME:
case ID_GET_FILE_RESUME_HASH:
case ID_GET_FILE_METADATA_ONLY:
case ID_GET_FILE_WITH_METADATA:
{
errorcode = 0;
std::string s_filename;
if(data->getStr(&s_filename)==false)
break;
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -2 ("+ ident+")", LL_DEBUG);
return false;
}
#endif
int64 metadata_id = 0;
int64 folder_items=0;
if(id==ID_GET_FILE_METADATA_ONLY)
{
char c_version;
if(!data->getChar(&c_version))
{
break;
}
if(c_version!=0)
{
break;
}
if(!data->getVarInt(&folder_items))
{
break;
}
if(!data->getVarInt(&metadata_id))
{
break;
}
}
with_hashes = id==ID_GET_FILE_RESUME_HASH;
bool with_sparse = false;
if(id==ID_GET_FILE_WITH_METADATA)
{
char c_version;
if(!data->getChar(&c_version))
{
break;
}
if(c_version!=0)
{
break;
}
char c_with_hash;
if(!data->getChar(&c_with_hash))
{
break;
}
with_hashes = c_with_hash!=0;
if(!data->getVarInt(&metadata_id))
{
break;
}
char c_with_sparse_handling = 0;
if (!data->getChar(&c_with_sparse_handling))
{
break;
}
with_sparse = c_with_sparse_handling != 0;
}
_i64 start_offset=0;
bool resumed=data->getInt64(&start_offset);
bool is_script = false;
if(next(s_filename, 0, "SCRIPT|"))
{
s_filename = s_filename.substr(7);
is_script=true;
}
std::string o_filename=s_filename;
if(id!=ID_GET_FILE_METADATA_ONLY)
{
if(!is_script)
{
Log("Sending file (normal) "+o_filename+" metadata_id="+convert(metadata_id), LL_DEBUG);
}
else
{
Log("Sending script output (normal) "+o_filename + " metadata_id=" + convert(metadata_id), LL_DEBUG);
}
}
else
{
Log("Sending meta data of "+o_filename +" metadata_id=" + convert(metadata_id) , LL_DEBUG);
if (metadata_id == 0)
{
Log("Trying to send metadata bug metadata_id is zero", LL_ERROR);
return false;
}
}
bool allow_exec;
std::string filename=map_file(o_filename, ident, allow_exec, NULL);
if (is_script)
{
filename = FileServ::getRedirectedFn(filename);
}
Log("Mapped name: "+filename, LL_DEBUG);
if(filename.empty())
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -1", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -1", LL_DEBUG);
break;
}
if( with_hashes )
{
hash_func.init();
}
bool has_file_extents = false;
std::vector<SExtent> file_extents;
#ifdef _WIN32
if(!is_script)
{
if(filename.size()>=2 && filename[0]=='\\' && filename[1]=='\\' )
{
if(filename.size()<3 || filename[2]!='?')
{
filename="\\\\?\\UNC"+filename.substr(1);
}
}
else
{
filename = "\\\\?\\"+filename;
}
}
if(bufmgr==NULL && !is_script)
{
bufmgr=new fileserv::CBufMgr(NBUFFERS,READSIZE);
}
#endif
if (!is_script
&& metadata_id != 0
&& FileServ::hasReadError(filename))
{
FileServ::clearReadErrorFile(filename);
char ch = ID_READ_ERROR;
int rc = SendInt(&ch, 1);
if (rc == SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send ID_READ_ERROR", LL_DEBUG);
return false;
}
Log("Info: Returning read error instead of sending file \""+filename+"\"", LL_DEBUG);
break;
}
if(is_script)
{
ScopedPipeFileUser pipe_file_user;
IFile* file = NULL;
bool sent_metadata = false;
if(next(s_filename, 0, "urbackup/FILE_METADATA|"))
{
file = PipeSessions::getFile(o_filename, pipe_file_user, std::string(), ident, NULL, NULL, resumed);
}
else if (next(s_filename, 0, "urbackup/TAR|"))
{
std::string server_token = getbetween("|", "|", s_filename);
std::string tar_fn = getafter("urbackup/TAR|" + server_token + "|", s_filename);
std::string map_res = map_file(tar_fn, ident, allow_exec, NULL);
if (!map_res.empty() && allow_exec)
{
file = PipeSessions::getFile(tar_fn, pipe_file_user, server_token, ident, &sent_metadata, NULL, true);
}
}
else if(allow_exec)
{
size_t tpos = filename.find_last_of('|');
std::string server_token;
if (tpos != std::string::npos)
{
server_token = filename.substr(tpos + 1);
}
bool tar_file = false;
file = PipeSessions::getFile(filename, pipe_file_user, server_token, ident, NULL, &tar_file, resumed);
if (metadata_id != 0)
{
PipeSessions::transmitFileMetadata(
getuntil("|", s_filename), getDummyMetadata(getuntil("|", s_filename), folder_items, metadata_id, tar_file), server_token, ident);
}
}
if(!file)
{
if (sent_metadata)
{
CWData data;
data.addUChar(ID_FILESIZE);
data.addUInt64(0);
int rc = SendInt(data.getDataPtr(), data.getDataSize());
if (rc == SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: SendSize (script metadata)", LL_DEBUG);
return false;
}
break;
}
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open script", LL_DEBUG);
break;
}
else
{
bool b = sendFullFile(file, start_offset, with_hashes);
if(!b)
{
Log("Sending script "+o_filename+" not finished yet", LL_INFO);
}
else
{
Log("Sent script "+o_filename, LL_INFO);
}
break;
}
}
else if(metadata_id!=0 && next(s_filename, 0, "clientdl"))
{
PipeSessions::transmitFileMetadata(filename,
s_filename, ident, ident, folder_items, metadata_id);
}
else if(metadata_id!=0 && s_filename.find("|")!=std::string::npos)
{
PipeSessions::transmitFileMetadata(filename,
getafter("|",s_filename), getuntil("|", s_filename), ident, folder_items, metadata_id);
}
ScopedShareActive scoped_share_active(o_filename);
#ifndef LINUX
DWORD extra_flags = 0;
if (id == ID_GET_FILE_METADATA_ONLY)
{
extra_flags = FILE_FLAG_OPEN_REPARSE_POINT;
}
#ifndef BACKUP_SEM
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OVERLAPPED|FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#else
if (backup_semantics)
extra_flags |= FILE_FLAG_BACKUP_SEMANTICS;
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING,
FILE_FLAG_OVERLAPPED|FILE_FLAG_SEQUENTIAL_SCAN|extra_flags, NULL);
#endif
if(hFile == INVALID_HANDLE_VALUE)
{
std::string openerr = os_last_error_str();
#ifdef CHECK_BASE_PATH
std::string share_name = getuntil("/",o_filename);
if(!share_name.empty())
{
bool allow_exec;
std::string basePath=map_file(share_name+"/", ident, allow_exec, NULL);
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG);
return false;
}
Log("Info: Base dir lost", LL_DEBUG);
break;
}
}
#endif
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Could not open file "+ filename+". "+ openerr, metadata_id != 0 ? LL_ERROR : LL_INFO);
break;
}
currfilepart=0;
sendfilepart=0;
sent_bytes=start_offset;
LARGE_INTEGER filesize;
GetFileSizeEx(hFile, &filesize);
curr_filesize=filesize.QuadPart;
if (curr_filesize == 0)
{
with_sparse = false;
}
int64 n_sparse_extents;
if (with_sparse)
{
int64 new_fsize = getFileExtents(curr_filesize, n_sparse_extents, file_extents, has_file_extents, start_offset);
if (new_fsize >= 0)
{
curr_filesize = new_fsize;
}
else
{
Log("Getting extents of file \"" + filename + "\" failed", LL_WARNING);
start_offset = sent_bytes;
}
}
else
{
has_file_extents = false;
}
next_checkpoint= sent_bytes + c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
BY_HANDLE_FILE_INFORMATION fi;
if(GetFileInformationByHandle(hFile, &fi)==TRUE &&
fi.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
{
curr_filesize = 0;
}
CWData data;
if (with_sparse && n_sparse_extents>0)
{
data.addUChar(ID_FILESIZE_AND_EXTENTS);
}
else
{
data.addUChar(ID_FILESIZE);
}
data.addUInt64(curr_filesize);
if (with_sparse && n_sparse_extents>0)
{
data.addInt64(n_sparse_extents);
}
int rc=SendInt(data.getDataPtr(), data.getDataSize());
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: SendSize", LL_DEBUG);
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
return false;
}
if (with_sparse && n_sparse_extents>0)
{
if (!sendExtents(file_extents, filesize.QuadPart, n_sparse_extents))
{
Log("Error sending sparse extents", LL_DEBUG);
CloseHandle(hFile);
hFile = INVALID_HANDLE_VALUE;
return false;
}
}
else
{
has_file_extents = false;
}
if(curr_filesize==0 || id==ID_GET_FILE_METADATA_ONLY)
{
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
break;
}
assert(!(GetFileAttributesW(Server->ConvertToWchar(filename).c_str()) & FILE_ATTRIBUTE_DIRECTORY));
size_t extent_pos = 0;
while (has_file_extents
&& file_extents[extent_pos].offset<=start_offset)
{
if (file_extents[extent_pos].offset + file_extents[extent_pos].size > start_offset)
{
break;
}
else if (extent_pos + 1 < file_extents.size())
{
++extent_pos;
}
else
{
char ch = ID_READ_ERROR;
int rc = SendInt(&ch, 1);
if (rc == SOCKET_ERROR)
{
return false;
}
Log("Beyond file extents when resuming " + filename +
" from position "+convert(start_offset)+".", LL_ERROR);
extent_pos = std::string::npos;
break;
}
}
if (extent_pos == std::string::npos)
{
break;
}
for(_i64 i=start_offset;i<filesize.QuadPart && !stopped;)
{
bool last = false;
_u32 toread;
if (has_file_extents)
{
assert(file_extents[extent_pos].offset + file_extents[extent_pos].size > i);
assert(file_extents[extent_pos].offset <= i);
toread = static_cast<_u32>((std::min)(file_extents[extent_pos].offset + file_extents[extent_pos].size - i, static_cast<int64>(READSIZE)));
if (i + toread == file_extents[extent_pos].offset + file_extents[extent_pos].size
&& extent_pos + 1 >= file_extents.size())
{
last = true;
}
}
else
{
toread = READSIZE;
}
if(i+toread>=filesize.QuadPart)
{
last=true;
Log("Reading last file part", LL_DEBUG);
}
while(bufmgr->nfreeBufffer()==0 && !stopped)
{
int rc;
SleepEx(0,true);
rc=SendData();
if(rc==-1)
{
Log("Error: Send failed in file loop -1", LL_DEBUG);
stopped=true;
}
else if(rc==0)
{
SleepEx(1,true);
}
}
if(errorcode!=0)
{
Log("Error occurred while reading from file \""+ filename+"\" (code: "+convert(errorcode)+").", LL_ERROR);
FileServ::callErrorCallback(o_filename, filename, i, "code: " + convert(errorcode));
stopped=true;
}
if( !stopped && hFile!=INVALID_HANDLE_VALUE)
{
if(!ReadFilePart(hFile, i, last, toread))
{
unsigned int ec = (unsigned int)GetLastError();
Log("Reading from file failed. Last error is "+convert(ec), LL_ERROR);
FileServ::callErrorCallback(o_filename, filename, i, "code: " + convert(ec));
stopped=true;
}
}
if(FileServ::isPause() )
{
DWORD starttime=GetTickCount();
while(GetTickCount()-starttime<5000 && !stopped)
{
SleepEx(500,true);
int rc=SendData();
if(rc==-1)
{
Log("Error: Send failed in file pause loop -2", LL_DEBUG);
stopped=true;
}
}
}
i += toread;
if (has_file_extents
&& i == file_extents[extent_pos].offset + file_extents[extent_pos].size)
{
++extent_pos;
if (extent_pos < file_extents.size())
{
i = file_extents[extent_pos].offset;
}
else
{
break;
}
}
}
while(bufmgr->nfreeBufffer()!=NBUFFERS && !stopped)
{
SleepEx(0,true);
if (errorcode != 0)
{
Log("Error occurred while reading from file \"" + filename + "\" (code: " + convert(errorcode) + ") -2.", LL_ERROR);
FileServ::callErrorCallback(o_filename, filename, filesize.QuadPart, "code: " + convert(errorcode));
stopped = true;
}
int rc;
rc=SendData();
if( rc==2 && bufmgr->nfreeBufffer()!=NBUFFERS )
{
Log("Error: File end and not all Buffers are free!-1", LL_WARNING);
}
if(rc==-1)
{
Log("Error: Send failed in off file loop -3", LL_DEBUG);
stopped=true;
}
else if(rc==0)
{
SleepEx(1,true);
}
}
while(stopped && bufmgr->nfreeBufffer()!=NBUFFERS)
{
SleepEx(100, true);
for(size_t i=0;i<t_send.size();++i)
{
if(t_send[i]->delbuf)
{
bufmgr->releaseBuffer(t_send[i]->delbufptr);
}
delete t_send[i];
}
t_send.clear();
for (size_t i = 0; i < t_unsend.size(); ++i)
{
bufmgr->releaseBuffer(t_unsend[i]->buffer);
delete t_unsend[i];
}
t_unsend.clear();
}
if( !stopped )
{
Log("Closed file.", LL_DEBUG);
if(hFile!=INVALID_HANDLE_VALUE)
{
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
}
}
#else //LINUX
if(id==ID_GET_FILE_METADATA_ONLY || isDirectory(filename))
{
CWData data;
data.addUChar(ID_FILESIZE);
data.addUInt64(little_endian(static_cast<uint64>(0)));
int rc=SendInt(data.getDataPtr(), data.getDataSize() );
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send file size", LL_DEBUG);
}
break;
}
int flags = O_RDONLY | O_LARGEFILE;
#if defined(O_CLOEXEC)
flags |= O_CLOEXEC;
#endif
#if defined(O_NOATIME)
if(backup_semantics)
flags |= O_NOATIME;
#endif
hFile=open64(filename.c_str(), flags);
if(hFile == INVALID_HANDLE_VALUE)
{
#ifdef CHECK_BASE_PATH
std::string share_name = getuntil("/",o_filename);
if(!share_name.empty())
{
bool allow_exec;
std::string basePath=map_file(share_name+"/", ident, allow_exec, NULL);
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST", LL_DEBUG);
return false;
}
Log("Info: Base dir lost", LL_DEBUG);
break;
}
}
#endif
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open file", LL_DEBUG);
break;
}
currfilepart=0;
sendfilepart=0;
struct stat64 stat_buf;
fstat64(hFile, &stat_buf);
off64_t filesize=stat_buf.st_size;
curr_filesize=filesize;
int64 send_filesize = curr_filesize;
int64 n_sparse_extents;
if (with_sparse)
{
send_filesize = getFileExtents(curr_filesize, n_sparse_extents, file_extents, has_file_extents, start_offset);
if (send_filesize<0)
{
Log("Error resetting file pointer", LL_DEBUG);
CloseHandle(hFile);
hFile = INVALID_HANDLE_VALUE;
return false;
}
}
CWData data;
if (has_file_extents)
{
data.addUChar(ID_FILESIZE_AND_EXTENTS);
}
else
{
data.addUChar(ID_FILESIZE);
}
data.addUInt64(little_endian(static_cast<uint64>(send_filesize)));
if (has_file_extents)
{
data.addInt64(n_sparse_extents);
}
int rc=SendInt(data.getDataPtr(), data.getDataSize() );
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: SendSize", LL_DEBUG);
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
return false;
}
if (has_file_extents)
{
if (!sendSparseExtents(file_extents))
{
Log("Error sending sparse extents", LL_DEBUG);
CloseHandle(hFile);
hFile = INVALID_HANDLE_VALUE;
return false;
}
}
if(filesize==0 || id==ID_GET_FILE_METADATA_ONLY)
{
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
break;
}
off64_t foffset=start_offset;
unsigned int s_bsize=8192;
if( !with_hashes )
{
s_bsize=32768;
next_checkpoint=curr_filesize;
}
else
{
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
}
if((clientpipe!=NULL || with_hashes) && foffset>0)
{
if(lseek64(hFile, foffset, SEEK_SET)!=foffset)
{
Log("Error: Seeking in file failed (5043)", LL_ERROR);
CloseHandle(hFile);
return false;
}
}
std::vector<char> buf;
buf.resize(s_bsize);
bool has_error=false;
size_t extent_pos = 0;
if (has_file_extents)
{
while (extent_pos < file_extents.size()
&& foffset >= file_extents[extent_pos].offset+ file_extents[extent_pos].size)
{
++extent_pos;
}
}
bool last_sent_hash = false;
while (foffset < filesize)
{
if (has_file_extents)
{
bool is_finished = false;
while (extent_pos < file_extents.size()
&& foffset >= file_extents[extent_pos].offset)
{
foffset += file_extents[extent_pos].size;
next_checkpoint += file_extents[extent_pos].size;
if (next_checkpoint>curr_filesize)
next_checkpoint = curr_filesize;
if (clientpipe != NULL || with_hashes)
{
off64_t rc = lseek64(hFile, foffset, SEEK_SET);
if (rc != foffset)
{
Log("Error seeking after sparse extent", LL_ERROR);
CloseHandle(hFile);
return false;
}
}
if (foffset >= filesize
&& last_sent_hash)
{
is_finished = true;
}
++extent_pos;
}
if (is_finished)
{
break;
}
}
size_t count=(std::min)((size_t)s_bsize, (size_t)(next_checkpoint-foffset));
if (has_file_extents)
{
if (extent_pos < file_extents.size())
{
count = static_cast<size_t>((std::min)(file_extents[extent_pos].offset - foffset, static_cast<int64>(count)));
}
}
if( clientpipe==NULL && !with_hashes && count>0 )
{
#if defined(__APPLE__) || defined(__FreeBSD__)
ssize_t rc=sendfile64(int_socket, hFile, foffset, count, reinterpret_cast<off_t*>(&count));
if(rc==0)
{
foffset+=count;
rc=count;
}
#else
ssize_t rc=sendfile64(int_socket, hFile, &foffset, count);
#endif
if(rc<0)
{
Log("Error: Reading and sending from file failed. Errno: "+convert(errno), LL_DEBUG);
FileServ::callErrorCallback(o_filename, filename, foffset, "code: " + convert(errno));
CloseHandle(hFile);
return false;
}
else if(rc==0 && foffset<filesize && errno == 0) //other process made the file smaller
{
memset(buf.data(), 0, s_bsize);
while(foffset<filesize)
{
rc=SendInt(buf.data(), (std::min)((size_t)s_bsize, (size_t)(next_checkpoint-foffset)));
if(rc==SOCKET_ERROR)
{
Log("Error: Sending data failed");
CloseHandle(hFile);
return false;
}
}
}
}
else
{
if (count > 0)
{
ssize_t rc = read(hFile, buf.data(), count);
if (rc == 0 && rc < count && errno == 0) //other process made the file smaller
{
memset(buf.data() + rc, 0, count - rc);
rc = count;
}
else if (rc < 0)
{
Log("Error: Reading from file failed. Errno: " + convert(errno), LL_DEBUG);
FileServ::callErrorCallback(o_filename, filename, foffset, "code: " + convert(errno));
CloseHandle(hFile);
return false;
}
rc = SendInt(buf.data(), rc);
if (rc == SOCKET_ERROR)
{
Log("Error: Sending data failed");
CloseHandle(hFile);
return false;
}
else if (with_hashes)
{
hash_func.update((unsigned char*)buf.data(), rc);
}
foffset += rc;
last_sent_hash = false;
}
if(with_hashes && foffset==next_checkpoint)
{
hash_func.finalize();
rc = SendInt((char*)hash_func.raw_digest_int(), 16);
if (rc == SOCKET_ERROR)
{
Log("Error: Sending hash failed");
CloseHandle(hFile);
return false;
}
next_checkpoint+=c_checkpoint_dist;
if(next_checkpoint>curr_filesize)
next_checkpoint=curr_filesize;
hash_func.init();
last_sent_hash = true;
}
}
if(FileServ::isPause() )
{
Sleep(500);
}
}
CloseHandle(hFile);
hFile=INVALID_HANDLE_VALUE;
#endif
}break;
case ID_GET_FILE_BLOCKDIFF:
{
bool b=GetFileBlockdiff(data, false);
if(!b)
return false;
}break;
case ID_GET_FILE_BLOCKDIFF_WITH_METADATA:
{
bool b=GetFileBlockdiff(data, true);
if(!b)
return false;
}break;
case ID_BLOCK_REQUEST:
{
if(state==CS_BLOCKHASH)
{
Handle_ID_BLOCK_REQUEST(data);
}
}break;
/*case ID_GET_FILE_HASH_AND_METADATA:
{
if(!GetFileHashAndMetadata(data))
{
return false;
}
}break;*/
case ID_INFORM_METADATA_STREAM_END:
{
if(!InformMetadataStreamEnd(data))
{
return false;
}
} break;
case ID_SCRIPT_FINISH:
{
if(!FinishScript(data))
{
return false;
}
} break;
case ID_STOP_PHASH:
{
if (!StopPhash(data))
{
return false;
}
} break;
case ID_FREE_SERVER_FILE:
{
if (chunk_send_thread_ticket != ILLEGAL_THREADPOOL_TICKET)
{
queueChunk(SChunk(ID_FREE_SERVER_FILE));
}
}break;
case ID_FLUSH_SOCKET:
{
if (chunk_send_thread_ticket != ILLEGAL_THREADPOOL_TICKET)
{
queueChunk(SChunk(ID_FLUSH_SOCKET));
}
else
{
Server->Log("Received flush.", LL_DEBUG);
if (!clientpipe->Flush(CLIENT_TIMEOUT * 1000))
{
Server->Log("Error flushing output socket", LL_INFO);
}
}
} break;
}
}
if( stopped )
return false;
else
return true;
}
#ifndef LINUX
void ProcessReadData( SLPData *ldata )
{
for(DWORD i=0;i<ldata->bsize;i+=SENDSIZE)
{
SSendData *sdata=new SSendData;
if( i+SENDSIZE >=ldata->bsize )
{
sdata->bsize=ldata->bsize-i;
sdata->delbufptr=ldata->buffer;
sdata->delbuf=true;
sdata->last=ldata->last;
}
else
{
sdata->bsize=SENDSIZE;
sdata->delbufptr=NULL;
sdata->delbuf=false;
sdata->last=false;
}
sdata->buffer=&ldata->buffer[i];
ldata->t_send->push_back(sdata);
}
}
void CALLBACK FileIOCompletionRoutine(DWORD dwErrorCode,DWORD dwNumberOfBytesTransfered,LPOVERLAPPED lpOverlapped)
{
SLPData *ldata=(SLPData*)lpOverlapped->hEvent;
if(dwErrorCode!=ERROR_SUCCESS)
{
*ldata->errorcode = dwErrorCode;
}
if(*ldata->errorcode==0
&& dwNumberOfBytesTransfered > 0)
{
ldata->bsize=dwNumberOfBytesTransfered;
if( *ldata->sendfilepart!=ldata->filepart )
{
Log("Packets out of order.... shifting", LL_DEBUG);
ldata->t_unsend->push_back(ldata);
delete lpOverlapped;
return;
}
ProcessReadData(ldata);
++(*ldata->sendfilepart);
if( ldata->t_unsend->size()>0 )
{
bool refresh=true;
while( refresh==true )
{
refresh=false;
for(size_t i=0;i<ldata->t_unsend->size();++i)
{
SLPData *pdata=(*ldata->t_unsend)[i];
if( *pdata->sendfilepart==pdata->filepart )
{
Log("Using shifted packet...", LL_DEBUG);
ProcessReadData(pdata);
++(*pdata->sendfilepart);
refresh=true;
ldata->t_unsend->erase( ldata->t_unsend->begin()+i );
delete pdata;
break;
}
}
}
}
delete ldata;
}
else
{
ldata->bufmgr->releaseBuffer(ldata->buffer);
Log( "Info: Chunk size=0", LL_DEBUG);
delete ldata;
}
delete lpOverlapped;
}
#endif
#ifndef LINUX
bool CClientThread::ReadFilePart(HANDLE hFile, _i64 offset, bool last, _u32 toread)
{
LPOVERLAPPED overlap=new OVERLAPPED;
//memset(overlap, 0, sizeof(OVERLAPPED) );
overlap->Offset=(DWORD)offset;
overlap->OffsetHigh=(DWORD)(offset>>32);
SLPData *ldata=new SLPData;
ldata->buffer=bufmgr->getBuffer();
ldata->bufmgr = bufmgr;
if( ldata->buffer==NULL )
{
Log("Error: No Free Buffer", LL_DEBUG);
Log("Info: Free Buffers="+convert(bufmgr->nfreeBufffer()), LL_DEBUG );
delete ldata;
delete overlap;
return true;
}
ldata->t_send=&t_send;
ldata->t_unsend=&t_unsend;
ldata->last=last;
ldata->filepart=currfilepart;
ldata->sendfilepart=&sendfilepart;
ldata->errorcode=&errorcode;
overlap->hEvent=ldata;
BOOL b=ReadFileEx(hFile, ldata->buffer, toread, overlap, FileIOCompletionRoutine);
++currfilepart;
if( b==FALSE)
{
Log("Error: Can't start reading from File", LL_DEBUG);
bufmgr->releaseBuffer(ldata->buffer);
delete ldata;
delete overlap;
return false;
}
else
{
return true;
}
}
#else
bool CClientThread::ReadFilePart(HANDLE hFile, _i64 offset, bool last, _u32 toread)
{
return false;
}
#endif
int CClientThread::SendData()
{
if( t_send.size()==0 )
return 0;
SSendData* ldata=t_send[0];
_i32 ret;
ret=clientpipe->isWritable(CLIENT_TIMEOUT*1000)?1:0;
if(ret < 1)
{
Log("Client Timeout occured.", LL_DEBUG);
if( ldata->delbuf )
{
bufmgr->releaseBuffer(ldata->delbufptr);
ldata->delbuf=false;
}
t_send.erase( t_send.begin() );
delete ldata;
return -1;
}
else
{
if( ldata->bsize>0 )
{
unsigned int sent=0;
while(sent<ldata->bsize)
{
_i32 ts;
if(with_hashes)
ts=(std::min)((unsigned int)(next_checkpoint-sent_bytes), ldata->bsize-sent);
else
ts=ldata->bsize;
_i32 rc=SendInt(&ldata->buffer[sent], ts);
bool has_error = false;
if( rc==SOCKET_ERROR )
{
has_error = true;
}
else if(with_hashes)
{
hash_func.update((unsigned char*)&ldata->buffer[sent], ts);
}
sent+=ts;
sent_bytes+=ts;
if(with_hashes
&& !has_error)
{
if(next_checkpoint-sent_bytes==0)
{
hash_func.finalize();
rc = SendInt((char*)hash_func.raw_digest_int(), 16);
if (rc == SOCKET_ERROR)
{
has_error = true;
}
else
{
next_checkpoint += c_checkpoint_dist;
if (next_checkpoint > curr_filesize)
next_checkpoint = curr_filesize;
hash_func.init();
}
}
}
if (has_error)
{
int err;
#ifdef _WIN32
err = WSAGetLastError();
#else
err = errno;
#endif
Log("SOCKET_ERROR in SendData(). BSize: " + convert(ldata->bsize) + " WSAGetLastError: " + convert(err), LL_DEBUG);
if (ldata->delbuf)
{
bufmgr->releaseBuffer(ldata->delbufptr);
ldata->delbuf = false;
}
t_send.erase(t_send.begin());
delete ldata;
return -1;
}
}
}
else
{
Log("ldata is null", LL_DEBUG);
}
if( ldata->delbuf==true )
{
bufmgr->releaseBuffer( ldata->delbufptr );
ldata->delbuf=false;
}
if( ldata->last==true )
{
Log("Info: File End", LL_DEBUG);
if( t_send.size() > 1 )
{
Log("Error: Senddata exceeds 1", LL_DEBUG);
}
for(size_t i=0;i<t_send.size();++i)
{
if( t_send[i]->delbuf==true )
{
bufmgr->releaseBuffer( t_send[i]->buffer );
}
delete t_send[i];
}
t_send.clear();
for (size_t i = 0; i < t_unsend.size(); ++i)
{
bufmgr->releaseBuffer(t_unsend[i]->buffer);
delete t_unsend[i];
}
t_unsend.clear();
return 2;
}
t_send.erase( t_send.begin() );
delete ldata;
return 1;
}
}
void CClientThread::ReleaseMemory(void)
{
#ifdef _WIN32
Log("Deleting Memory...", LL_DEBUG);
if(bufmgr!=NULL)
{
while( bufmgr->nfreeBufffer()!=NBUFFERS )
{
while( SleepEx(1000,true)!=0 );
for(size_t i=0;i<t_send.size();++i)
{
if( t_send[i]->delbuf==true )
bufmgr->releaseBuffer( t_send[i]->delbufptr );
delete t_send[i];
}
t_send.clear();
for (size_t i = 0; i < t_unsend.size(); ++i)
{
bufmgr->releaseBuffer(t_unsend[i]->buffer);
delete t_unsend[i];
}
t_unsend.clear();
}
}
Log("done.", LL_DEBUG);
#endif
}
void CClientThread::CloseThread(HANDLE hFile)
{
StopThread();
}
bool CClientThread::isStopped(void)
{
return stopped;
}
void CClientThread::StopThread(void)
{
stopped=true;
clientpipe->shutdown();
Log("Client thread stopped", LL_DEBUG);
}
bool CClientThread::isKillable(void)
{
return killable;
}
bool CClientThread::GetFileBlockdiff(CRData *data, bool with_metadata)
{
std::string s_filename;
if(data->getStr(&s_filename)==false)
return false;
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -2", LL_DEBUG);
return false;
}
#endif
int64 metadata_id=0;
bool with_sparse = false;
if(with_metadata)
{
char c_version;
if(!data->getChar(&c_version))
{
return false;
}
if(c_version!=0)
{
return false;
}
if(!data->getVarInt(&metadata_id))
{
return false;
}
char c_with_sparse;
if (!data->getChar(&c_with_sparse))
{
return false;
}
with_sparse = c_with_sparse == 1;
}
bool is_script=false;
if(next(s_filename, 0, "SCRIPT|"))
{
is_script=true;
s_filename = s_filename.substr(7);
}
std::string o_filename=s_filename;
_i64 start_offset=0;
data->getInt64(&start_offset);
_i64 curr_hash_size=0;
data->getInt64(&curr_hash_size);
_i64 requested_filesize=-1;
data->getInt64(&requested_filesize);
unsigned char flags1 = 0;
data->getUChar(&flags1);
bool resumed = false;
if (flags1 & 1)
{
resumed = true;
}
Log("Sending file (chunked) "+o_filename, LL_DEBUG);
bool allow_exec;
std::string filename=map_file(o_filename, ident, allow_exec, NULL);
if (is_script)
{
filename = FileServ::getRedirectedFn(filename);
}
Log("Mapped name: "+filename, LL_DEBUG);
state=CS_BLOCKHASH;
if(filename.empty())
{
queueChunk(SChunk(ID_BASE_DIR_LOST));
Log("Info: Base dir lost -1", LL_DEBUG);
return true;
}
hash_func.init();
#ifdef _WIN32
if(!is_script)
{
if(filename.size()>=2 && filename[0]=='\\' && filename[1]=='\\' )
{
if(filename.size()<3 || filename[2]!='?')
{
filename="\\\\?\\UNC"+filename.substr(1);
}
}
else
{
filename = "\\\\?\\"+filename;
}
}
#endif
if (!is_script
&& metadata_id != 0
&& FileServ::hasReadError(filename))
{
FileServ::clearReadErrorFile(filename);
queueChunk(SChunk(ID_READ_ERROR));
Log("Info: Returning read error instead of file \""+filename+"\"", LL_DEBUG);
return true;
}
ScopedShareActive scoped_share_active;
std::unique_ptr<ScopedPipeFileUser> pipe_file_user;
IFile* srv_file = NULL;
IFileServ::CbtHashFileInfo cbt_hash_file_info;
IFileServ::IMetadataCallback* metadata_callback = NULL;
if(is_script)
{
pipe_file_user.reset(new ScopedPipeFileUser);
if (next(s_filename, 0, "urbackup/TAR|"))
{
std::string server_token = getbetween("|", "|", s_filename);
std::string tar_fn = getafter("urbackup/TAR|" + server_token + "|", s_filename);
std::string map_res = map_file(tar_fn, ident, allow_exec, NULL);
if (!map_res.empty() && allow_exec)
{
srv_file = PipeSessions::getFile(tar_fn, *pipe_file_user, server_token, ident, NULL, NULL, true);
}
}
else if(allow_exec)
{
size_t tpos = filename.find_last_of('|');
std::string server_token;
if (tpos != std::string::npos)
{
server_token = filename.substr(tpos + 1);
}
bool tar_file = false;
srv_file = PipeSessions::getFile(filename, *pipe_file_user, server_token, ident, NULL, &tar_file, resumed);
if (metadata_id != 0)
{
PipeSessions::transmitFileMetadata(
getuntil("|", s_filename), getDummyMetadata(getuntil("|", s_filename), 0, metadata_id, tar_file), server_token, ident);
}
}
if(srv_file==NULL)
{
queueChunk(SChunk(ID_COULDNT_OPEN));
Log("Info: Couldn't open script", LL_DEBUG);
return true;
}
}
else
{
if(metadata_id!=0 && next(s_filename, 0, "clientdl"))
{
metadata_callback = PipeSessions::transmitFileMetadata(filename,
s_filename, ident, ident, 0, metadata_id);
}
else if(metadata_id!=0 && s_filename.find("|"))
{
PipeSessions::transmitFileMetadata(filename,
getafter("|",s_filename), getuntil("|", s_filename), ident, 0, metadata_id);
}
scoped_share_active.reset(s_filename);
#ifdef _WIN32
#ifndef BACKUP_SEM
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#else
DWORD flags = FILE_FLAG_SEQUENTIAL_SCAN;
if (backup_semantics)
flags |= FILE_FLAG_BACKUP_SEMANTICS;
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, flags, NULL);
#endif
#else //_WIN32
int flags = O_RDONLY | O_LARGEFILE;
#if defined(O_CLOEXEC)
flags |= O_CLOEXEC;
#endif
#if defined(O_NOATIME)
if(backup_semantics)
flags |= O_NOATIME;
#endif
hFile=open64(filename.c_str(), flags);
#endif //_WIN32
if(hFile == INVALID_HANDLE_VALUE)
{
std::string errstr = os_last_error_str();
#ifdef CHECK_BASE_PATH
std::string share_name = getuntil("/",o_filename);
if(!share_name.empty())
{
bool allow_exec;
std::string basePath=map_file(share_name+"/", ident, allow_exec, NULL);
if(!isDirectory(basePath))
{
queueChunk(SChunk(ID_BASE_DIR_LOST));
Log("Info: Base dir lost", LL_DEBUG);
return true;
}
}
#endif
queueChunk(SChunk(ID_COULDNT_OPEN));
Log("Could not open file "+filename+". " + errstr, metadata_id != 0 ? LL_ERROR : LL_INFO);
return true;
}
}
currfilepart=0;
sendfilepart=0;
sent_bytes=0;
if(!is_script)
{
#ifdef _WIN32
LARGE_INTEGER filesize;
GetFileSizeEx(hFile, &filesize);
curr_filesize=filesize.QuadPart;
#else
struct stat64 stat_buf;
fstat64(hFile, &stat_buf);
curr_filesize=stat_buf.st_size;
#endif
if (curr_filesize > c_checkpoint_dist)
{
map_file(o_filename, ident, allow_exec, &cbt_hash_file_info);
if (cbt_hash_file_info.cbt_hash_file == NULL
&& metadata_callback != NULL)
{
int64 offset, length;
IFile* metadata_f = metadata_callback->getMetadata("f" + s_filename, NULL, &offset, &length, NULL, true);
if (metadata_f != NULL)
{
cbt_hash_file_info.cbt_hash_file = metadata_f;
cbt_hash_file_info.metadata_offset = offset;
cbt_hash_file_info.metadata_size = length;
}
}
}
}
else
{
curr_filesize = srv_file->Size();
}
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize && curr_filesize>0)
next_checkpoint=curr_filesize;
if(!is_script)
{
srv_file = Server->openFileFromHandle((void*)hFile, filename);
if(srv_file==NULL)
{
if (cbt_hash_file_info.cbt_hash_file != NULL
&& cbt_hash_file_info.metadata_offset != -1)
{
Server->destroy(cbt_hash_file_info.cbt_hash_file);
}
CloseHandle(hFile);
queueChunk(SChunk(ID_COULDNT_OPEN));
Log("Info: Couldn't open file from handle", LL_ERROR);
return true;
}
}
SChunk chunk;
chunk.update_file = srv_file;
chunk.startpos = curr_filesize;
chunk.hashsize = curr_hash_size;
chunk.requested_filesize = requested_filesize;
chunk.pipe_file_user = pipe_file_user.get();
chunk.with_sparse = is_script ? false : with_sparse;
chunk.s_filename = s_filename;
chunk.cbt_hash_file_info = cbt_hash_file_info;
pipe_file_user.release();
hFile=INVALID_HANDLE_VALUE;
chunk.share_active_gen = scoped_share_active.release();
queueChunk(chunk);
return true;
}
bool CClientThread::Handle_ID_BLOCK_REQUEST(CRData *data)
{
SChunk chunk;
chunk.update_file = NULL;
bool b=data->getInt64(&chunk.startpos);
if(!b)
return false;
b=data->getChar(&chunk.transfer_all);
if(!b)
return false;
if(data->getLeft()==big_hash_size+small_hash_size*(c_checkpoint_dist/c_small_hash_dist))
{
memcpy(chunk.big_hash, data->getCurrDataPtr(), big_hash_size);
data->incrementPtr(big_hash_size);
memcpy(chunk.small_hash, data->getCurrDataPtr(), small_hash_size*(c_checkpoint_dist/c_small_hash_dist));
}
else if(chunk.transfer_all==0)
{
return false;
}
queueChunk(chunk);
return true;
}
bool CClientThread::getNextChunk(SChunk *chunk, bool has_error)
{
IScopedLock lock(mutex);
if(has_error)
{
clientpipe->shutdown();
return false;
}
while(next_chunks.empty() && state==CS_BLOCKHASH)
{
waiting_for_chunk = true;
cond->wait(&lock);
waiting_for_chunk = false;
}
if(!next_chunks.empty() && state==CS_BLOCKHASH)
{
*chunk=next_chunks.front();
next_chunks.pop();
return true;
}
else
{
return false;
}
}
void CClientThread::queueChunk(const SChunk& chunk )
{
if(chunk_send_thread_ticket==ILLEGAL_THREADPOOL_TICKET)
{
if(mutex==NULL)
{
mutex=Server->createMutex();
cond=Server->createCondition();
}
next_chunks.push(chunk);
chunk_send_thread_ticket=Server->getThreadPool()->execute(new ChunkSendThread(this), "filesrv: chunk send");
}
else
{
IScopedLock lock(mutex);
next_chunks.push(chunk);
cond->notify_all();
}
}
bool CClientThread::GetFileHashAndMetadata( CRData* data )
{
std::string s_filename;
if(data->getStr(&s_filename)==false)
return false;
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -hash", LL_DEBUG);
return false;
}
#endif
std::string o_filename=s_filename;
Log("Calculating hash of file "+o_filename, LL_DEBUG);
bool allow_exec;
std::string filename=map_file(o_filename, ident, allow_exec, NULL);
Log("Mapped name: "+filename, LL_DEBUG);
if(filename.empty())
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -hash2", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -hash", LL_DEBUG);
return true;
}
#ifdef _WIN32
if(filename.size()>=2 && filename[0]=='\\' && filename[1]=='\\' )
{
if(filename.size()<3 || filename[2]!='?')
{
filename="\\\\?\\UNC"+filename.substr(1);
}
}
else
{
filename = "\\\\?\\"+filename;
}
#endif
#ifdef _WIN32
#ifndef BACKUP_SEM
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
#else
DWORD flags = FILE_FLAG_SEQUENTIAL_SCAN;
if (backup_semantics)
flags |= FILE_FLAG_BACKUP_SEMANTICS;
hFile=CreateFileW(Server->ConvertToWchar(filename).c_str(), FILE_READ_DATA, FILE_SHARE_READ, NULL, OPEN_EXISTING, flags, NULL);
#endif
#else //_WIN32
int flags = O_RDONLY | O_LARGEFILE;
#if defined(O_CLOEXEC)
flags |= O_CLOEXEC;
#endif
#if defined(O_NOATIME)
if(backup_semantics)
flags |= O_NOATIME;
#endif
hFile=open64(filename.c_str(), flags);
#endif //_WIN32
if(hFile == INVALID_HANDLE_VALUE)
{
#ifdef CHECK_BASE_PATH
std::string share_name = getuntil("/",o_filename);
bool allow_exec;
std::string basePath=map_file(share_name + "/", ident, allow_exec, NULL);
if(!isDirectory(basePath))
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -hash", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -hash", LL_DEBUG);
return false;
}
#endif
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN -hash", LL_DEBUG);
return false;
}
Log("Info: Couldn't open file -hash", LL_DEBUG);
return false;
}
std::unique_ptr<IFile> tf(Server->openFileFromHandle((void*)hFile, filename));
if(tf.get()==NULL)
{
Log("Could not open file from handle -hash", LL_ERROR);
return false;
}
unsigned int read;
std::vector<char> buffer;
buffer.resize(32768);
sha512_ctx ctx;
sha512_init(&ctx);
while( (read=tf->Read(&buffer[0], static_cast<_u32>(buffer.size())))>0 )
{
sha512_update(&ctx, reinterpret_cast<const unsigned char*>(&buffer[0]), read);
}
std::string dig;
dig.resize(64);
sha512_final(&ctx, reinterpret_cast<unsigned char*>(&dig[0]));
CWData send_data;
send_data.addUChar(ID_FILE_HASH_AND_METADATA);
send_data.addUShort(0);
send_data.addString(dig);
send_data.addString(FileServFactory::getPermissionCallback()->getPermissions(filename));
send_data.addInt64(tf->Size());
SFile file_metadata = getFileMetadata(filename);
if(file_metadata.name.empty())
{
Log("Could not get metadata of file", LL_DEBUG);
return false;
}
send_data.addInt64(file_metadata.last_modified);
send_data.addInt64(file_metadata.created);
unsigned short send_data_size=static_cast<unsigned short>(send_data.getDataSize());
memcpy(send_data.getDataPtr()+1, &send_data_size, sizeof(send_data_size) );
int rc=SendInt(send_data.getDataPtr(), send_data.getDataSize());
if(rc==SOCKET_ERROR)
{
Log("Socket error while sending hash", LL_DEBUG);
return false;
}
return true;
}
bool CClientThread::sendFullFile(IFile* file, _i64 start_offset, bool with_hashes)
{
curr_filesize = file->Size();
if (start_offset != 0)
{
Log("Sending pipe file from offset " + convert(start_offset), LL_DEBUG);
}
if (!file->Seek(start_offset))
{
Log("Error: Seeking in file failed (5044) to " + convert(start_offset) + " file size is " + convert(file->Size()), LL_ERROR);
char ch = ID_BASE_DIR_LOST;
int rc = SendInt(&ch, 1);
if (rc == SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -3", LL_DEBUG);
}
return false;
}
CWData data;
data.addUChar(ID_FILESIZE);
data.addUInt64(little_endian(static_cast<uint64>(curr_filesize)));
int rc=SendInt(data.getDataPtr(), data.getDataSize(), true);
if(rc==SOCKET_ERROR)
{
return false;
}
if(curr_filesize==0)
{
return true;
}
unsigned int s_bsize=8192;
if(!with_hashes)
{
s_bsize=32768;
if(curr_filesize>=0)
{
next_checkpoint=curr_filesize;
}
else
{
next_checkpoint = LLONG_MAX;
}
}
else
{
next_checkpoint=start_offset+c_checkpoint_dist;
if(next_checkpoint>curr_filesize && curr_filesize>0)
next_checkpoint=curr_filesize;
}
std::vector<char> buf;
buf.resize(s_bsize);
int64 foffset = start_offset;
bool is_eof=false;
int64 last_flush_foffset = foffset;
while( foffset < curr_filesize || (curr_filesize==-1 && !is_eof) )
{
size_t count=(std::min)((size_t)s_bsize, (size_t)(next_checkpoint-foffset));
bool has_error = false;
_u32 rc = file->Read(foffset, &buf[0], static_cast<_u32>(count), &has_error);
if(rc==0 && curr_filesize==-1 && file->Size()!=-1)
{
is_eof=true;
}
if(rc>=0 && rc<count && curr_filesize!=-1 && !has_error)
{
memset(&buf[rc], 0, count-rc);
rc=static_cast<_u32>(count);
}
else if(has_error)
{
Log("Error: Reading from file failed.", LL_DEBUG);
return false;
}
bool curr_flush = rc<count;
if (foffset - last_flush_foffset>1 * 1024 * 1024)
{
curr_flush = true;
last_flush_foffset = foffset;
}
rc=SendInt(buf.data(), rc, curr_flush);
if(rc==SOCKET_ERROR)
{
Log("Error: Sending data failed");
return false;
}
else if(with_hashes)
{
hash_func.update((unsigned char*)buf.data(), rc);
}
foffset+=rc;
if(with_hashes && foffset==next_checkpoint)
{
hash_func.finalize();
rc=SendInt((char*)hash_func.raw_digest_int(), 16, curr_flush);
if (rc == SOCKET_ERROR)
{
Log("Error: Sending hash failed");
return false;
}
next_checkpoint+=c_checkpoint_dist;
if(next_checkpoint>curr_filesize && curr_filesize>0)
next_checkpoint=curr_filesize;
hash_func.init();
}
if(FileServ::isPause() )
{
Sleep(500);
}
}
if(curr_filesize==-1)
{
if(!clientpipe->Flush(CLIENT_TIMEOUT*1000))
{
Server->Log("Error flushing output socket (3)", LL_INFO);
}
//script needs one reconnect
clientpipe->shutdown();
}
return curr_filesize!=-1;
}
bool CClientThread::InformMetadataStreamEnd( CRData * data )
{
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -hash", LL_DEBUG);
return false;
}
#endif
std::string token;
data->getStr(&token);
PipeSessions::metadataStreamEnd(token);
char ch = ID_PONG;
int rc = SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Sending data failed (InformMetadataStreamEnd)");
return false;
}
if(!clientpipe->Flush(CLIENT_TIMEOUT*1000))
{
Server->Log("Error flushing output socket (2)", LL_INFO);
}
return true;
}
bool CClientThread::StopPhash(CRData * data)
{
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if (!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -hash", LL_DEBUG);
return false;
}
#endif
std::string token;
data->getStr(&token);
std::string fn;
data->getStr(&fn);
PipeSessions::phashEnd(token, fn);
char ch = ID_PONG;
int rc = SendInt(&ch, 1);
if (rc == SOCKET_ERROR)
{
Log("Error: Sending data failed (InformMetadataStreamEnd)");
return false;
}
if (!clientpipe->Flush(CLIENT_TIMEOUT * 1000))
{
Server->Log("Error flushing output socket (2)", LL_INFO);
}
return true;
}
bool CClientThread::FinishScript( CRData * data )
{
#ifdef CHECK_IDENT
std::string ident;
data->getStr(&ident);
if(!FileServ::checkIdentity(ident, is_tunneled))
{
Log("Identity check failed -FinishScript", LL_DEBUG);
return false;
}
#endif
std::string s_filename;
data->getStr(&s_filename);
if(next(s_filename, 0, "SCRIPT|"))
{
s_filename = s_filename.substr(7);
}
else
{
return false;
}
Log("Finishing script "+s_filename, LL_DEBUG);
bool allow_exec;
std::string filename=map_file(s_filename, ident, allow_exec, NULL);
filename = FileServ::getRedirectedFn(filename);
Log("Mapped name: "+filename, LL_DEBUG);
if(filename.empty())
{
char ch=ID_BASE_DIR_LOST;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send BASE_DIR_LOST -4", LL_DEBUG);
return false;
}
Log("Info: Base dir lost -2", LL_DEBUG);
clientpipe->Flush(CLIENT_TIMEOUT*1000);
return false;
}
ScopedPipeFileUser pipe_file_user;
IFile* file = NULL;
std::string f_name;
bool sent_metadata = false;
if(next(s_filename, 0, "urbackup/FILE_METADATA|"))
{
f_name = s_filename;
file = PipeSessions::getFile(s_filename, pipe_file_user, std::string(), ident, NULL, NULL, true);
}
else if (next(s_filename, 0, "urbackup/TAR|"))
{
std::string server_token = getbetween("|", "|", s_filename);
f_name = getafter("urbackup/TAR|" + server_token + "|", s_filename);
std::string map_res = map_file(f_name, ident, allow_exec, NULL);
if (!map_res.empty() && allow_exec)
{
file = PipeSessions::getFile(f_name, pipe_file_user, server_token, ident, &sent_metadata, NULL, true);
}
}
else if(allow_exec)
{
f_name = filename;
file = PipeSessions::getFile(filename, pipe_file_user, std::string(), ident, NULL, NULL, true);
}
bool ret=false;
if(!file && !sent_metadata)
{
char ch=ID_COULDNT_OPEN;
int rc=SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Socket Error - DBG: Send COULDNT OPEN", LL_DEBUG);
return false;
}
Log("Info: Couldn't open script", LL_DEBUG);
}
else
{
pipe_file_user.reset(NULL);
PipeSessions::removeFile(f_name);
char ch = ID_PONG;
int rc = SendInt(&ch, 1);
if(rc==SOCKET_ERROR)
{
Log("Error: Sending data failed (FinishScript)");
return false;
}
ret=true;
}
if(!clientpipe->Flush(CLIENT_TIMEOUT*1000))
{
Server->Log("Error flushing output socket (2)", LL_INFO);
ret=false;
}
return ret;
}
std::string CClientThread::getDummyMetadata(std::string output_fn, int64 folder_items, int64 metadata_id, bool is_dir)
{
CWData data;
data.addChar(ID_METADATA_V1);
_u32 fn_start = data.getDataSize();
std::string type = "f";
if (is_dir)
{
type = "d";
}
data.addString(type + output_fn);
data.addUInt(urb_adler32(urb_adler32(0, NULL, 0), data.getDataPtr() + fn_start, static_cast<_u32>(data.getDataSize()) - fn_start));
_u32 common_start = data.getDataSize();
data.addUInt(0);
data.addChar(1);
data.addVarInt(Server->getTimeSeconds());
data.addVarInt(Server->getTimeSeconds());
data.addVarInt(0);
data.addVarInt(folder_items);
data.addVarInt(metadata_id);
std::unique_ptr<IFileServ::ITokenCallback> token_callback(FileServ::newTokenCallback());
std::string ttokens;
if (token_callback.get() != NULL)
{
ttokens = token_callback->translateTokens(0, 0, 0000400);
}
if (ttokens.empty())
{
//allow to all
CWData token_info;
token_info.addChar(0);
token_info.addVarInt(0);
ttokens = std::string(token_info.getDataPtr(), token_info.getDataSize());
}
data.addString(ttokens);
_u32 common_metadata_size = little_endian(static_cast<_u32>(data.getDataSize() - common_start - sizeof(_u32)));
memcpy(data.getDataPtr() + common_start, &common_metadata_size, sizeof(common_metadata_size));
data.addUInt(urb_adler32(urb_adler32(0, NULL, 0), data.getDataPtr() + common_start, static_cast<_u32>(data.getDataSize()) - common_start));
_u32 os_start = data.getDataSize();
#ifdef _WIN32
data.addUInt(0);
data.addChar(1);
data.addUInt(0); //atributes
data.addVarInt(os_to_windows_filetime(Server->getTimeSeconds())); //creation time
data.addVarInt(0); //last access time
data.addVarInt(os_to_windows_filetime(Server->getTimeSeconds())); //modify time
data.addVarInt(os_to_windows_filetime(Server->getTimeSeconds())); //ctime
_u32 stat_data_size = little_endian(static_cast<_u32>(data.getDataSize()) - os_start - sizeof(_u32));
memcpy(data.getDataPtr() + os_start, &stat_data_size, sizeof(stat_data_size));
data.addChar(0);
#else
data.addUInt(0);
struct stat64 buf = {};
buf.st_mtime = Server->getTimeSeconds();
buf.st_ctime = buf.st_mtime;
if (is_dir)
{
buf.st_mode = 0000400 | 0040000;
}
else
{
buf.st_mode = 0000400 | 0100000;
}
serialize_stat_buf(buf, std::string(), data);
_u32 stat_data_size = little_endian(static_cast<_u32>(data.getDataSize() - os_start - sizeof(_u32)));
memcpy(data.getDataPtr() + os_start, &stat_data_size, sizeof(stat_data_size));
data.addInt64(0);
#endif
data.addUInt(urb_adler32(urb_adler32(0, NULL, 0), data.getDataPtr() + os_start, static_cast<_u32>(data.getDataSize()) - os_start));
return std::string(data.getDataPtr(), data.getDataSize());
}
int64 CClientThread::getFileExtents(int64 fsize, int64& n_sparse_extents, std::vector<SExtent>& file_extents, bool& has_file_extents, int64& start_offset)
{
int64 real_fsize = 0;
n_sparse_extents = 0;
#ifdef _WIN32
FILE_ALLOCATED_RANGE_BUFFER query_range;
query_range.FileOffset.QuadPart = 0;
query_range.Length.QuadPart = fsize;
int64 last_sparse_pos = 0;
std::vector<FILE_ALLOCATED_RANGE_BUFFER> res_buffer;
res_buffer.resize(1000);
while (true)
{
DWORD output_bytes;
BOOL b = DeviceIoControl(hFile, FSCTL_QUERY_ALLOCATED_RANGES,
&query_range, sizeof(query_range), res_buffer.data(), static_cast<DWORD>(res_buffer.size()*sizeof(FILE_ALLOCATED_RANGE_BUFFER)),
&output_bytes, NULL);
bool more_data = (!b && GetLastError() == ERROR_MORE_DATA);
if (more_data || b)
{
size_t res_size = output_bytes / sizeof(FILE_ALLOCATED_RANGE_BUFFER);
size_t start_off = file_extents.size();
file_extents.resize(start_off + res_size);
for (size_t i = start_off; i < start_off + res_size; ++i)
{
file_extents[i] = SExtent(res_buffer[i - start_off].FileOffset.QuadPart, res_buffer[i - start_off].Length.QuadPart);
if (file_extents[i].offset != last_sparse_pos)
{
if (last_sparse_pos <= start_offset)
{
start_offset += file_extents[i].offset - last_sparse_pos;
}
++n_sparse_extents;
}
last_sparse_pos = file_extents[i].offset + file_extents[i].size;
if (i + 1 >= start_off + res_size)
{
query_range.FileOffset.QuadPart = res_buffer[i - start_off].FileOffset.QuadPart + res_buffer[i - start_off].Length.QuadPart;
query_range.Length.QuadPart = fsize - query_range.FileOffset.QuadPart;
}
real_fsize += file_extents[i].size;
}
if (!more_data)
{
has_file_extents = true;
if (last_sparse_pos != fsize)
{
if (last_sparse_pos <= start_offset)
{
start_offset += fsize - last_sparse_pos;
}
++n_sparse_extents;
}
return real_fsize;
}
}
if (!b && !more_data)
{
has_file_extents = false;
return -1;
}
}
#elif SEEK_HOLE
off64_t last_sparse_pos = 0;
has_file_extents = false;
real_fsize = fsize;
bool seek_suc = false;
while (true)
{
int64 sparse_hole_start = lseek64(hFile, last_sparse_pos, SEEK_HOLE);
if (sparse_hole_start == -1)
{
break;
}
seek_suc = true;
if (sparse_hole_start == fsize)
{
break;
}
last_sparse_pos = lseek64(hFile, sparse_hole_start, SEEK_DATA);
if (last_sparse_pos == -1)
{
if (sparse_hole_start <= start_offset)
{
start_offset += fsize - sparse_hole_start;
}
real_fsize -= fsize - sparse_hole_start;
file_extents.push_back(SExtent(sparse_hole_start, fsize - sparse_hole_start));
has_file_extents = true;
break;
}
if (sparse_hole_start == last_sparse_pos)
{
break;
}
if (sparse_hole_start <= start_offset)
{
start_offset += last_sparse_pos - sparse_hole_start;
}
real_fsize -= last_sparse_pos - sparse_hole_start;
file_extents.push_back(SExtent(sparse_hole_start, last_sparse_pos - sparse_hole_start));
has_file_extents = true;
}
if (seek_suc)
{
off64_t rc = lseek64(hFile, 0, SEEK_SET);
if (rc != 0)
{
return -1;
}
}
n_sparse_extents = file_extents.size();
#else
has_file_extents = false;
real_fsize = fsize;
#endif
return real_fsize;
}
bool CClientThread::sendExtents(const std::vector<SExtent>& file_extents, int64 fsize, int64 n_sparse_extents)
{
std::vector<int64> send_buf;
send_buf.resize(n_sparse_extents*2+2);
int64 last_sparse_pos = 0;
size_t curr_sparse_ext = 0;
MD5 hash_func;
for (size_t i = 0; i < file_extents.size(); ++i)
{
if (file_extents[i].offset != last_sparse_pos)
{
send_buf[curr_sparse_ext * 2 + 0] = last_sparse_pos;
send_buf[curr_sparse_ext * 2 + 1] = file_extents[i].offset - last_sparse_pos;
++curr_sparse_ext;
}
last_sparse_pos = file_extents[i].offset + file_extents[i].size;
}
if (last_sparse_pos != fsize)
{
send_buf[curr_sparse_ext * 2 + 0] = last_sparse_pos;
send_buf[curr_sparse_ext * 2 + 1] = fsize - last_sparse_pos;
++curr_sparse_ext;
}
assert(curr_sparse_ext == n_sparse_extents);
hash_func.update(reinterpret_cast<unsigned char*>(send_buf.data()), static_cast<_u32>(n_sparse_extents * 2 * sizeof(int64)));
hash_func.finalize();
memcpy(&send_buf[n_sparse_extents * 2], hash_func.raw_digest_int(), 2 * sizeof(int64));
int rc = SendInt(reinterpret_cast<char*>(send_buf.data()), send_buf.size()*sizeof(int64));
return rc != SOCKET_ERROR;
}
bool CClientThread::sendSparseExtents(const std::vector<SExtent>& file_extents)
{
std::vector<int64> send_buf;
send_buf.resize(file_extents.size() * 2 + 2);
int64 last_sparse_pos = 0;
size_t curr_sparse_ext = 0;
MD5 hash_func;
for (size_t i = 0; i < file_extents.size(); ++i)
{
send_buf[i * 2 + 0] = little_endian(file_extents[i].offset);
send_buf[i * 2 + 1] = little_endian(file_extents[i].size);
}
hash_func.update(reinterpret_cast<unsigned char*>(send_buf.data()), static_cast<_u32>(file_extents.size() * 2 * sizeof(int64)));
hash_func.finalize();
memcpy(&send_buf[file_extents.size() * 2], hash_func.raw_digest_int(), 2 * sizeof(int64));
int rc = SendInt(reinterpret_cast<char*>(send_buf.data()), send_buf.size()*sizeof(int64));
return rc != SOCKET_ERROR;
}
Reference in New Issue View Git Blame Copy Permalink