diff --git a/src/libsync/bulkpropagatorjob.cpp b/src/libsync/bulkpropagatorjob.cpp index 2d06298fe1..ae3c940627 100644 --- a/src/libsync/bulkpropagatorjob.cpp +++ b/src/libsync/bulkpropagatorjob.cpp @@ -70,6 +70,7 @@ Q_LOGGING_CATEGORY(lcBulkPropagatorJob, "nextcloud.sync.propagator.bulkupload", BulkPropagatorJob::BulkPropagatorJob(OwncloudPropagator *propagator, const std::deque &items) : PropagatorJob(propagator) , _items(items) + , _currentBatchSize(batchSize) { _filesToUpload.reserve(batchSize); _pendingChecksumFiles.reserve(batchSize); @@ -83,7 +84,7 @@ bool BulkPropagatorJob::scheduleSelfOrChild() _state = Running; - for(auto i = 0; i < batchSize && !_items.empty(); ++i) { + for(auto i = 0; i < _currentBatchSize && !_items.empty(); ++i) { const auto currentItem = _items.front(); _items.pop_front(); _pendingChecksumFiles.insert(currentItem->_file); @@ -107,6 +108,29 @@ bool BulkPropagatorJob::scheduleSelfOrChild() return _items.empty() && _filesToUpload.empty(); } +bool BulkPropagatorJob::handleBatchSize() +{ + // no error, no batch size to change + if (_finalStatus == SyncFileItem::Success || _finalStatus == SyncFileItem::NoStatus) { + qCDebug(lcBulkPropagatorJob) << "No error, no need to change the bulk upload batch size!"; + return true; + } + + // change batch size before trying it again + const auto halfBatchSize = batchSize / 2; + + // we already tried to upload with half of the batch size + if(_currentBatchSize == halfBatchSize) { + qCDebug(lcBulkPropagatorJob) << "There was another error, stop syncing now!"; + return false; + } + + // try to upload with half of the batch size + _currentBatchSize = halfBatchSize; + qCDebug(lcBulkPropagatorJob) << "There was an error, sync again with bulk upload batch size cut to half!"; + return true; +} + PropagatorJob::JobParallelism BulkPropagatorJob::parallelism() const { return PropagatorJob::JobParallelism::FullParallelism; @@ -254,13 +278,16 @@ void BulkPropagatorJob::checkPropagationIsDone() // just wait for the other job to finish. return; } - - qCInfo(lcBulkPropagatorJob) << "final status" << _finalStatus; - emit finished(_finalStatus); - propagator()->scheduleNextJob(); } else { - scheduleSelfOrChild(); + if (handleBatchSize()) { + scheduleSelfOrChild(); + return; + } } + + qCInfo(lcBulkPropagatorJob) << "final status" << _finalStatus; + emit finished(_finalStatus); + propagator()->scheduleNextJob(); } void BulkPropagatorJob::slotComputeTransmissionChecksum(SyncFileItemPtr item, @@ -374,6 +401,9 @@ void BulkPropagatorJob::slotPutFinishedOneFile(const BulkUploadItem &singleFile, singleFile._item->_status = SyncFileItem::Success; + // upload succeeded, so remove from black list + propagator()->removeFromBulkUploadBlackList(singleFile._item->_file); + // Check the file again post upload. // Two cases must be considered separately: If the upload is finished, // the file is on the server and has a changed ETag. In that case, diff --git a/src/libsync/bulkpropagatorjob.h b/src/libsync/bulkpropagatorjob.h index 3e4303461f..7d6c7425f5 100644 --- a/src/libsync/bulkpropagatorjob.h +++ b/src/libsync/bulkpropagatorjob.h @@ -155,6 +155,8 @@ private: void checkPropagationIsDone(); + bool handleBatchSize(); + std::deque _items; QVector _jobs; /// network jobs that are currently in transit @@ -166,6 +168,7 @@ private: qint64 _sentTotal = 0; SyncFileItem::Status _finalStatus = SyncFileItem::Status::NoStatus; + int _currentBatchSize = 0; }; } diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 2cfbc9566e..9855a2321f 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -427,8 +427,6 @@ std::unique_ptr OwncloudPropagator::createUploadJob(S job->setDeleteExisting(deleteExisting); - removeFromBulkUploadBlackList(item->_file); - return job; } @@ -1338,8 +1336,9 @@ void PropagatorCompositeJob::finalize() { // The propagator will do parallel scheduling and this could be posted // multiple times on the event loop, ignore the duplicate calls. - if (_state == Finished) + if (_state == Finished) { return; + } _state = Finished; emit finished(_hasError == SyncFileItem::NoStatus ? SyncFileItem::Success : _hasError);