Add js_scenario param (#9)

Author: kirby81

Makefile

@@ -8,7 +8,7 @@ lint:  ## Lint code
 	flake8 --config flake8 scrapingbee/ tests/ setup.py
 
 test:  ## Run tests
-	pytest tests/
+	python -m pytest tests/
 
 build:  ## Build a binary wheel and a source tarball
 	python setup.py sdist bdist_wheel

README.md

@@ -42,14 +42,17 @@ Signup to ScrapingBee to [get your API key](https://app.scrapingbee.com/account/
         'extract_rules': {'title': 'h1'},
         # Wrap response in JSON
         'json_response': False,
-        # JavaScript snippet to execute (clicking on a button, scrolling ...)
-        'js_snippet': '',
-        # Scrolling to the end of the page before returning your results
-        'js_scroll': False,
-        # The time to wait between each scroll	
-        'js_scroll_wait': 1000,
-        # The number of scrolls you want to make	
-        'js_scroll_count': 1,
+        # Interact with the webpage you want to scrape 
+        'json_scenario': {
+            "instructions": [
+                {"wait_for": "#slow_button"},
+                {"click": "#slow_button"},
+                {"scroll_x": 1000},
+                {"wait": 1000},
+                {"scroll_x": 1000},
+                {"wait": 1000},            
+            ]
+        },
         # Use premium proxies to bypass difficult to scrape websites (10-25 credits/request)
         'premium_proxy': False,
         # Execute JavaScript code with a Headless Browser (5 credits/request)

RELEASE.md

@@ -2,7 +2,7 @@
 
 A new package is automatically uploaded to PyPI when a new tag is pushed to Github. To release a new version follow the steps:
 
-1. Update the version number X.X.X in [setup.py](setup.py) and push the change.
+1. Update the version number X.X.X in [scrapingbee/__version__.py](scrapingbee/__version__.py) and push the change.
 
 2. Create a tag with the same version number.
 

scrapingbee/__version__.py

@@ -1 +1 @@
-__version__ = '1.1.6'
+__version__ = '1.1.7'

scrapingbee/utils.py

@@ -25,11 +25,11 @@ def process_cookies(cookies: dict) -> str:
         return cookies
 
 
-def process_extract_rules(extract_rules: dict) -> str:
-    if isinstance(extract_rules, dict):
-        return urllib.parse.quote(json.dumps(extract_rules))
+def process_json_stringify_param(param: dict, param_name: str) -> str:
+    if isinstance(param, dict):
+        return urllib.parse.quote(json.dumps(param))
     else:
-        raise ValueError("extract_rules must be a dict or a stringified JSON")
+        raise ValueError(f"{param_name} must be a dict or a stringified JSON")
 
 
 def process_params(params: dict) -> dict:
@@ -44,7 +44,9 @@ def process_params(params: dict) -> dict:
         elif k == 'cookies':
             new_params[k] = process_cookies(v)
         elif k == 'extract_rules':
-            new_params[k] = process_extract_rules(v)
+            new_params[k] = process_json_stringify_param(v, 'extract_rules')
+        elif k == 'js_scenario':
+            new_params[k] = process_json_stringify_param(v, 'js_scenario')
         else:
             new_params[k] = v
     return new_params

tests/test_client.py

@@ -92,6 +92,27 @@ def test_get_with_extract_rules(mock_request, client):
     )
 
 
+@mock.patch('scrapingbee.client.request')
+def test_get_with_js_scenario(mock_request, client):
+    '''It should format the extract_rules and add them to the url'''
+    client.get('https://httpbin.org', params={
+        'js_scenario': {
+            'instructions': [
+                {"click": "#buttonId"}
+            ]
+        }
+    })
+
+    mock_request.assert_called_with(
+        'GET',
+        'https://app.scrapingbee.com/api/v1/'
+        '?api_key=API_KEY&url=https%3A//httpbin.org&'
+        'js_scenario=%7B%22instructions%22%3A%20%5B%7B%22click%22%3A%20%22%23buttonId%22%7D%5D%7D',
+        data=None,
+        headers=default_headers,
+    )
+
+
 @mock.patch('scrapingbee.client.request')
 def test_post(mock_request, client):
     '''It should make a POST request with some data'''

tests/test_utils.py

@@ -1,9 +1,9 @@
 from scrapingbee.utils import (
     process_url,
     process_js_snippet,
+    process_json_stringify_param,
     process_headers,
     process_cookies,
-    process_extract_rules,
     process_params,
     get_scrapingbee_url
 )
@@ -40,12 +40,22 @@ def test_process_cookies():
 
 def test_process_extract_rules():
     '''It should format extract_rules to a stringified JSON'''
-    output = process_extract_rules({
+    output = process_json_stringify_param({
         'title': '.title'
-    })
+    }, 'extract_rules')
     assert output == '%7B%22title%22%3A%20%22.title%22%7D'
 
 
+def test_process_js_scenario():
+    '''It should format js_scenario to a stringified JSON'''
+    output = process_json_stringify_param({
+        'instructions': [
+            {"click": "#buttonId"}
+        ]
+    }, 'js_scenario')
+    assert output == '%7B%22instructions%22%3A%20%5B%7B%22click%22%3A%20%22%23buttonId%22%7D%5D%7D'
+
+
 def test_process_params():
     '''It should keep boolean parameters'''
     output = process_params({'render_js': True})