Merge pull request #409 from scholarly-python-package/develop

Maintenance before v1.6.1

Author: arunkannawadi

.github/workflows/publish-to-pypi.yml

@@ -1,5 +1,7 @@
 name: Publish package to PyPI
-on: push
+on:
+  push:
+    branches: [main]
 
 jobs:
   build-n-publish:

.github/workflows/pythonpackage.yml

@@ -43,7 +43,12 @@ jobs:
       #  uses: typilus/typilus-action@v0.9
       - name: Install Chrome
         uses: browser-actions/setup-chrome@latest
-      - name: Test with unittest
+        continue-on-error: true
+        with:
+          fail_ci_if_error: false
+      - name: Run unittests
+        id: unittests
+        continue-on-error: true
         env:
           CONNECTION_METHOD: ${{ secrets.CONNECTION_METHOD }}
           PASSWORD: ${{ secrets.PASSWORD }}
@@ -52,20 +57,23 @@ jobs:
           SCRAPER_API_KEY: ${{ secrets.SCRAPER_API_KEY }}
         run: |
           coverage run -m unittest -v test_module.TestScholarly
-          coverage xml
-      - name: Validate the repository yaml for codecov
+      - name: Generate coverage report (macOS only)
         if:
-          "matrix.os == 'ubuntu-latest'"
-        run:
+          "matrix.os == 'macos-latest'"
+        run: |
           curl --data-binary @.github/.codecov.yml https://codecov.io/validate | head -n 1
-      - name: Upload code coverage
+          coverage xml
+      - name: Upload code coverage (macOS only)
         if:
-          "matrix.os != 'windows-latest'"
+          "matrix.os == 'macos-latest'"
         uses: codecov/codecov-action@v2
         with:
           directory: ./
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           files: ./coverage.xml
           flags: unittests
           name: codecov-umbrella
           verbose: true
+      - name: Check if unittests failed
+        if: "steps.unittests.outcome == 'failure'"
+        run: exit 1

scholarly/_proxy_generator.py

@@ -49,6 +49,7 @@ def __init__(self):
         self._tor_control_port = None
         self._tor_password = None
         self._session = None
+        self._webdriver = None
         self._TIMEOUT = 5
         self._new_session()
 
@@ -458,7 +459,10 @@ def _close_session(self):
         if self._session:
             self._session.close()
         if self._webdriver:
-            self._webdriver.quit()
+            try:
+                self._webdriver.quit()
+            except Exception as e:
+                self.logger.warning("Could not close webdriver cleanly: %s", e)
 
     def _fp_coroutine(self, timeout=1, wait_time=120):
         """A coroutine to continuosly yield free proxies

test_module.py

@@ -347,7 +347,7 @@ def test_search_author_id_filled(self):
         self.assertGreaterEqual(len(author['publications']), 179)
         pub = author['publications'][1]
         self.assertEqual(pub["citedby_url"],
-                         "https://scholar.google.com/scholar?oi=bibs&hl=en&cites=6983837810323809551")
+                         "https://scholar.google.com/scholar?oi=bibs&hl=en&cites=9976400141451962702")
 
     @unittest.skipIf(os.getenv("CONNECTION_METHOD") in {None, "none", "freeproxy"}, reason="No robust proxy setup")
     def test_search_pubs(self):
@@ -658,8 +658,8 @@ def test_download_mandates_csv(self):
             }
             agency_2020 = {
                 "US National Science Foundation": "86%",
-                "Department of Science & Technology, India": "47%",
-                "Swedish Research Council": "88%",
+                "Department of Science & Technology, India": "48%",
+                "Swedish Research Council": "89%",
                 "Swedish Research Council for Environment, Agricultural Sciences and Spatial Planning": "88%"
             }