diff --git a/lib/Service/ProvisioningService.php b/lib/Service/ProvisioningService.php
index 20ab7f4f..069f7431 100644
--- a/lib/Service/ProvisioningService.php
+++ b/lib/Service/ProvisioningService.php
@@ -762,13 +762,22 @@ public function provisionUserGroups(IUser $user, int $providerId, object $idToke
 			return null;
 		}
 
-		$userGroups = $this->groupManager->getUserGroupIds($user);
-		foreach ($userGroups as $groupGID) {
-			if (!in_array($groupGID, array_column($syncGroups, 'gid'))) {
-				if ($groupsWhitelistRegex && !preg_match($groupsWhitelistRegex, $groupGID)) {
-					continue;
+		if ($groupsWhitelistRegex) {
+			$userGroups = $this->groupManager->getUserGroups($user);
+			foreach ($userGroups as $group) {
+				if (!in_array($group->getGID(), array_column($syncGroups, 'gid'))) {
+					if (!preg_match($groupsWhitelistRegex, $group->getDisplayName())) {
+						continue;
+					}
+					$group->removeUser($user);
+				}
+			}
+		} else {
+			$userGroupIds = $this->groupManager->getUserGroupIds($user);
+			foreach ($userGroupIds as $groupGID) {
+				if (!in_array($groupGID, array_column($syncGroups, 'gid'))) {
+					$this->groupManager->get($groupGID)?->removeUser($user);
 				}
-				$this->groupManager->get($groupGID)?->removeUser($user);
 			}
 		}