From a135f1ebbbf444f7c210720c3061a4c2ff8f5b51 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 6 Dec 2013 15:46:00 +0100 Subject: [PATCH 01/12] add getAllGroups() that uses a paged search if available this circumvents server side search limits (active directory has a limit of 1000 by default) --- apps/user_ldap/group_ldap.php | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index cef9ca3c4cf..f455a68a34e 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -334,6 +334,40 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { return $ldap_groups; } + /** + * @brief get a list of all groups using a paged search + * @returns array with group names + * + * Returns a list with all groups + * Uses a paged search if available to override a + * server side search limit. + * (active directory has a limit of 1000 by default) + */ + public function getAllGroups($search = '', $max_groups= 100000, $chunksize=900) { + if(!$this->enabled) { + return array(); + } + if (! $this->access->connection->hasPagedResultSupport) { + return $this->getGroups($search); + } + $offset = 0; + $all_groups = array(); + while ($offset < $max_groups) { + $limit = min($chunksize, $max_groups - $offset); + $ldap_groups = $this->getGroups('', $limit, $offset); + $nread = count($ldap_groups); + \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$offset.' (limit: '.$limit.')', \OCP\Util::DEBUG); + if ($nread) { + $all_groups = array_merge($all_groups, $ldap_groups); + $offset += $nread; + } + if ($nread < $limit) { + break; + } + } + return $all_groups; + } + public function groupMatchesFilter($group) { return (strripos($group, $this->groupSearch) !== false); } From bacda81bf7bc20654954c3befa9cc3a95206ba0b Mon Sep 17 00:00:00 2001 From: root Date: Fri, 6 Dec 2013 15:51:19 +0100 Subject: [PATCH 02/12] use getAllGroup() in the updateGroups background job needed in active directory deployments with > 1000 groups --- apps/user_ldap/lib/jobs.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/lib/jobs.php b/apps/user_ldap/lib/jobs.php index 9b108da6331..2e5c440fd6b 100644 --- a/apps/user_ldap/lib/jobs.php +++ b/apps/user_ldap/lib/jobs.php @@ -41,7 +41,7 @@ class Jobs extends \OC\BackgroundJob\TimedJob { \OCP\Util::writeLog('user_ldap', 'Run background job "updateGroups"', \OCP\Util::DEBUG); $knownGroups = array_keys(self::getKnownGroups()); - $actualGroups = self::getGroupBE()->getGroups(); + $actualGroups = self::getGroupBE()->getAllGroups(); if(empty($actualGroups) && empty($knownGroups)) { \OCP\Util::writeLog('user_ldap', From a5813acb64c7679615a570d10933158ec054cec5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Mar 2014 17:23:04 +0100 Subject: [PATCH 03/12] add ldap_paging_size config option --- apps/user_ldap/lib/configuration.php | 5 ++++- apps/user_ldap/templates/settings.php | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/user_ldap/lib/configuration.php b/apps/user_ldap/lib/configuration.php index 612a623e910..cc68a41d739 100644 --- a/apps/user_ldap/lib/configuration.php +++ b/apps/user_ldap/lib/configuration.php @@ -77,6 +77,7 @@ class Configuration { 'ldapExpertUUIDGroupAttr' => null, 'lastJpegPhotoLookup' => null, 'ldapNestedGroups' => false, + 'ldapPagingSize' => null, ); /** @@ -344,6 +345,7 @@ class Configuration { 'has_memberof_filter_support' => 0, 'last_jpegPhoto_lookup' => 0, 'ldap_nested_groups' => 0, + 'ldap_paging_size' => 0, ); } @@ -395,7 +397,8 @@ class Configuration { 'ldap_expert_uuid_group_attr' => 'ldapExpertUUIDGroupAttr', 'has_memberof_filter_support' => 'hasMemberOfFilterSupport', 'last_jpegPhoto_lookup' => 'lastJpegPhotoLookup', - 'ldap_nested_groups' => 'ldapNestedGroups', + 'ldap_nested_groups' => 'ldapNestedGroups', + 'ldap_paging_size' => 'ldapPagingSize', ); return $array; } diff --git a/apps/user_ldap/templates/settings.php b/apps/user_ldap/templates/settings.php index 79c4ae224c3..a61d00bc159 100644 --- a/apps/user_ldap/templates/settings.php +++ b/apps/user_ldap/templates/settings.php @@ -37,6 +37,7 @@

+

t('Special Attributes'));?>

From 039f7b054a0ffb8ceda5feeb2c522adb737e6a7c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Mar 2014 18:02:27 +0100 Subject: [PATCH 04/12] merge functionality of getAllGroups into getGroups --- apps/user_ldap/group_ldap.php | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index f455a68a34e..c1008a5deda 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -299,9 +299,9 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { * @brief get a list of all groups * @returns array with group names * - * Returns a list with all groups + * Returns a list with all groups (used by getGroups) */ - public function getGroups($search = '', $limit = -1, $offset = 0) { + private function getGroupsChunk($search = '', $limit = -1, $offset = 0) { if(!$this->enabled) { return array(); } @@ -339,29 +339,32 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { * @returns array with group names * * Returns a list with all groups - * Uses a paged search if available to override a - * server side search limit. - * (active directory has a limit of 1000 by default) + * Uses a paged search if available to override a + * server side search limit. + * (active directory has a limit of 1000 by default) */ - public function getAllGroups($search = '', $max_groups= 100000, $chunksize=900) { + public function getGroups($search = '', $limit = -1, $offset = 0) { + $max_groups = 100000; // limit max results (just for safety reasons) if(!$this->enabled) { return array(); } - if (! $this->access->connection->hasPagedResultSupport) { - return $this->getGroups($search); + $pagingsize = $this->access->connection->ldapPagingSize; + if ((! $this->access->connection->hasPagedResultSupport) + || empty($pagingsize)) { + return $this->getGroupsChunk($search, $limit, $offset); } - $offset = 0; + $chunk_offset = $offset; $all_groups = array(); - while ($offset < $max_groups) { - $limit = min($chunksize, $max_groups - $offset); - $ldap_groups = $this->getGroups('', $limit, $offset); + while ($chunk_offset < $max_groups) { + $chunk_limit = min($pagingsize, $max_groups - $chunk_offset); + $ldap_groups = $this->getGroupsChunk('', $chunk_limit, $chunk_offset); $nread = count($ldap_groups); - \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$offset.' (limit: '.$limit.')', \OCP\Util::DEBUG); + \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$chunk_offset.' (limit: '.$chunk_limit.')', \OCP\Util::DEBUG); if ($nread) { $all_groups = array_merge($all_groups, $ldap_groups); - $offset += $nread; + $chunk_offset += $nread; } - if ($nread < $limit) { + if ($nread < $chunk_limit) { break; } } From eb59e63c3b031b9a80a24865beb41a94eaa3b246 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Mar 2014 19:47:49 +0100 Subject: [PATCH 05/12] revert updateGroups job to use getGroups() instead of getAllGroups() --- apps/user_ldap/lib/jobs.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/lib/jobs.php b/apps/user_ldap/lib/jobs.php index 2e5c440fd6b..9b108da6331 100644 --- a/apps/user_ldap/lib/jobs.php +++ b/apps/user_ldap/lib/jobs.php @@ -41,7 +41,7 @@ class Jobs extends \OC\BackgroundJob\TimedJob { \OCP\Util::writeLog('user_ldap', 'Run background job "updateGroups"', \OCP\Util::DEBUG); $knownGroups = array_keys(self::getKnownGroups()); - $actualGroups = self::getGroupBE()->getAllGroups(); + $actualGroups = self::getGroupBE()->getGroups(); if(empty($actualGroups) && empty($knownGroups)) { \OCP\Util::writeLog('user_ldap', From eb1e36182470b5aa71b4ecd2253479e81c388898 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Mar 2014 21:07:27 +0100 Subject: [PATCH 06/12] getGroups() $limit was not handled correctly --- apps/user_ldap/group_ldap.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index c1008a5deda..861636a763f 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -353,10 +353,15 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { || empty($pagingsize)) { return $this->getGroupsChunk($search, $limit, $offset); } + if ($limit > -1) { + $overall_limit = min($limit, 100000); + } else { + $overall_limit = 100000; + } $chunk_offset = $offset; $all_groups = array(); - while ($chunk_offset < $max_groups) { - $chunk_limit = min($pagingsize, $max_groups - $chunk_offset); + while ($chunk_offset < $overall_limit) { + $chunk_limit = min($pagingsize, $overall_limit - $chunk_offset); $ldap_groups = $this->getGroupsChunk('', $chunk_limit, $chunk_offset); $nread = count($ldap_groups); \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$chunk_offset.' (limit: '.$chunk_limit.')', \OCP\Util::DEBUG); From 4645d0a93a1dbf5c89b78d226d069de0dff13167 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 4 Mar 2014 21:12:47 +0100 Subject: [PATCH 07/12] getGroups(): fix overall max search limit for paged searches --- apps/user_ldap/group_ldap.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index 861636a763f..7efbc025ba8 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -344,7 +344,6 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { * (active directory has a limit of 1000 by default) */ public function getGroups($search = '', $limit = -1, $offset = 0) { - $max_groups = 100000; // limit max results (just for safety reasons) if(!$this->enabled) { return array(); } @@ -353,10 +352,11 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { || empty($pagingsize)) { return $this->getGroupsChunk($search, $limit, $offset); } + $max_groups = 100000; // limit max results (just for safety reasons) if ($limit > -1) { - $overall_limit = min($limit, 100000); + $overall_limit = min($limit, $max_groups); } else { - $overall_limit = 100000; + $overall_limit = $max_groups; } $chunk_offset = $offset; $all_groups = array(); From 84550fbfb472f31914fc155c4e9228f4c8ba6052 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 5 Mar 2014 10:42:35 +0100 Subject: [PATCH 08/12] fix another oversight... sigh.. --- apps/user_ldap/group_ldap.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index 7efbc025ba8..064631d789d 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -362,7 +362,7 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { $all_groups = array(); while ($chunk_offset < $overall_limit) { $chunk_limit = min($pagingsize, $overall_limit - $chunk_offset); - $ldap_groups = $this->getGroupsChunk('', $chunk_limit, $chunk_offset); + $ldap_groups = $this->getGroupsChunk($search, $chunk_limit, $chunk_offset); $nread = count($ldap_groups); \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$chunk_offset.' (limit: '.$chunk_limit.')', \OCP\Util::DEBUG); if ($nread) { From 7f8b04d3e0392849de42a2b834bc1213d4c6149f Mon Sep 17 00:00:00 2001 From: Alexander Bergolth Date: Thu, 13 Mar 2014 09:33:07 +0100 Subject: [PATCH 09/12] adapt coding style --- apps/user_ldap/group_ldap.php | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index 064631d789d..cb6d256e5cc 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -352,28 +352,28 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { || empty($pagingsize)) { return $this->getGroupsChunk($search, $limit, $offset); } - $max_groups = 100000; // limit max results (just for safety reasons) + $maxGroups = 100000; // limit max results (just for safety reasons) if ($limit > -1) { - $overall_limit = min($limit, $max_groups); + $overallLimit = min($limit, $maxGroups); } else { - $overall_limit = $max_groups; + $overallLimit = $maxGroups; } - $chunk_offset = $offset; - $all_groups = array(); - while ($chunk_offset < $overall_limit) { - $chunk_limit = min($pagingsize, $overall_limit - $chunk_offset); - $ldap_groups = $this->getGroupsChunk($search, $chunk_limit, $chunk_offset); - $nread = count($ldap_groups); - \OCP\Util::writeLog('user_ldap', 'getAllGroups('.$search.'): read '.$nread.' at offset '.$chunk_offset.' (limit: '.$chunk_limit.')', \OCP\Util::DEBUG); + $chunkOffset = $offset; + $allGroups = array(); + while ($chunkOffset < $overallLimit) { + $chunkLimit = min($pagingsize, $overallLimit - $chunkOffset); + $ldapGroups = $this->getGroupsChunk($search, $chunkLimit, $chunkOffset); + $nread = count($ldapGroups); + \OCP\Util::writeLog('user_ldap', 'getGroups('.$search.'): read '.$nread.' at offset '.$chunkOffset.' (limit: '.$chunkLimit.')', \OCP\Util::DEBUG); if ($nread) { - $all_groups = array_merge($all_groups, $ldap_groups); - $chunk_offset += $nread; + $allGroups = array_merge($allGroups, $ldapGroups); + $chunkOffset += $nread; } - if ($nread < $chunk_limit) { + if ($nread < $chunkLimit) { break; } } - return $all_groups; + return $allGroups; } public function groupMatchesFilter($group) { From c76d952dacd049971d128ec4061603e127de766e Mon Sep 17 00:00:00 2001 From: Alexander Bergolth Date: Thu, 13 Mar 2014 10:57:19 +0100 Subject: [PATCH 10/12] make getGroupsChunk() protected instead of private --- apps/user_ldap/group_ldap.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/group_ldap.php b/apps/user_ldap/group_ldap.php index cb6d256e5cc..4f2424d9531 100644 --- a/apps/user_ldap/group_ldap.php +++ b/apps/user_ldap/group_ldap.php @@ -301,7 +301,7 @@ class GROUP_LDAP extends BackendUtility implements \OCP\GroupInterface { * * Returns a list with all groups (used by getGroups) */ - private function getGroupsChunk($search = '', $limit = -1, $offset = 0) { + protected function getGroupsChunk($search = '', $limit = -1, $offset = 0) { if(!$this->enabled) { return array(); } From 0d9d70b51c8113713c5da05548c584ffc954162d Mon Sep 17 00:00:00 2001 From: Alexander Bergolth Date: Fri, 28 Mar 2014 14:02:19 +0100 Subject: [PATCH 11/12] change default ldap paging size to 500 --- apps/user_ldap/lib/configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/lib/configuration.php b/apps/user_ldap/lib/configuration.php index cc68a41d739..713fba8365b 100644 --- a/apps/user_ldap/lib/configuration.php +++ b/apps/user_ldap/lib/configuration.php @@ -345,7 +345,7 @@ class Configuration { 'has_memberof_filter_support' => 0, 'last_jpegPhoto_lookup' => 0, 'ldap_nested_groups' => 0, - 'ldap_paging_size' => 0, + 'ldap_paging_size' => 500, ); } From b1589392839a1481f351a43187b12de2de1a4076 Mon Sep 17 00:00:00 2001 From: Alexander Bergolth Date: Fri, 28 Mar 2014 14:20:01 +0100 Subject: [PATCH 12/12] adapt description of paging size configuration --- apps/user_ldap/templates/settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/user_ldap/templates/settings.php b/apps/user_ldap/templates/settings.php index a61d00bc159..95a888a31d4 100644 --- a/apps/user_ldap/templates/settings.php +++ b/apps/user_ldap/templates/settings.php @@ -37,7 +37,7 @@

-

+

t('Special Attributes'));?>