Merge branch 'main' into feature/add-analytics

Author: alallema

Pipfile

@@ -6,14 +6,14 @@ name = "pypi"
 [packages]
 Scrapy = "==2.9.0"
 selenium = "==4.9.1"
-meilisearch = "==0.25.0"
+meilisearch = "==0.26.0"
 requests-iap = "==0.2.0"
 python-keycloak-client = "==0.2.3"
 requests = "==2.31.0"
 
 [dev-packages]
 pytest = "==7.3.1"
-pylint = "==2.15.9"
+pylint = "==2.17.4"
 tox = "==4.5.2"
 tox-pipenv = "==1.10.1"
 importlib_metadata = {version = "*", markers="python_version < '3.8'"}

Pipfile.lock

@@ -1,3 +1,4 @@
+import builtins
 import re
 import os
 from selenium import webdriver
@@ -29,7 +30,7 @@ def init(config_original_content, js_render, user_agent):
             CHROMEDRIVER_PATH = os.environ.get('CHROMEDRIVER_PATH',
                                                "/usr/bin/chromedriver")
             if not os.path.isfile(CHROMEDRIVER_PATH):
-                raise Exception(
+                raise builtins.Exception(
                     f"Env CHROMEDRIVER_PATH='{CHROMEDRIVER_PATH}' is not a path to a file")
             driver = webdriver.Chrome(
                 CHROMEDRIVER_PATH,

scraper/src/config/browser_handler.py

@@ -1,5 +1,8 @@
 
 
+import builtins
+
+
 class ConfigValidator:
     config = None
 
@@ -18,52 +21,52 @@ def validate(self):
         # Start urls must be an array
         if self.config.start_urls and not isinstance(self.config.start_urls,
                                                      list):
-            raise Exception('start_urls should be list')
+            raise builtins.Exception('start_urls should be list')
 
         # Stop urls must be an array
         if self.config.stop_urls and not isinstance(self.config.stop_urls,
                                                     list):
-            raise Exception('stop_urls should be list')
+            raise builtins.Exception('stop_urls should be list')
 
         # Custom settings must be a dict
         if self.config.custom_settings and not isinstance(self.config.custom_settings,
                                                     dict):
-            raise Exception('custom_settings must be a dictionary')
+            raise builtins.Exception('custom_settings must be a dictionary')
 
         if self.config.js_render and not isinstance(self.config.js_render,
                                                     bool):
-            raise Exception('js_render should be boolean')
+            raise builtins.Exception('js_render should be boolean')
 
         # `js_wait` is set to 0s by default unless it is specified
         if self.config.js_wait and not isinstance(self.config.js_wait, int):
-            raise Exception('js_wait should be integer')
+            raise builtins.Exception('js_wait should be integer')
 
         if self.config.use_anchors and not isinstance(self.config.use_anchors,
                                                       bool):
-            raise Exception('use_anchors should be boolean')
+            raise builtins.Exception('use_anchors should be boolean')
 
         if self.config.sitemap_alternate_links and not isinstance(
                 self.config.sitemap_alternate_links, bool):
-            raise Exception('sitemap_alternate_links should be boolean')
+            raise builtins.Exception('sitemap_alternate_links should be boolean')
 
         if self.config.sitemap_urls_regexs and not self.config.sitemap_urls:
-            raise Exception(
+            raise builtins.Exception(
                 'You gave an regex to parse sitemap but you didn\'t provide a sitemap url')
 
         if self.config.sitemap_urls_regexs and not self.config.sitemap_urls:
             for regex in self.config.sitemap_urls_regex:
                 if not isinstance(regex, str):
-                    raise Exception(
+                    raise builtins.Exception(
                         'You gave an bad regex: ' + regex + ' must be a string')
 
         if self.config.force_sitemap_urls_crawling and not self.config.sitemap_urls:
-            raise Exception(
+            raise builtins.Exception(
                 'You want to force the sitemap crawling but you didn\'t provide a sitemap url')
 
         if not self.config.scrape_start_urls and not self.config.scrap_start_urls:
-            raise Exception(
+            raise builtins.Exception(
                 'Please use only the new variable name: scrape_start_urls')
 
         if self.config.nb_hits_max and not isinstance(self.config.nb_hits_max,
                                                       int):
-            raise Exception('nb_hits_max should be integer')
+            raise builtins.Exception('nb_hits_max should be integer')

scraper/src/config/config_validator.py

@@ -1,3 +1,4 @@
+import builtins
 from ..helpers import css_to_xpath
 
 
@@ -27,7 +28,7 @@ def _parse_selectors_set(config_selectors):
             # Type
             if 'type' in selectors_set[key]:
                 if selectors_set[key]['type'] not in ['xpath', 'css']:
-                    raise Exception(
+                    raise builtins.Exception(
                         selectors_set[key][
                             'type'] + 'is not a good selector type, it should be `xpath` or `css`')
             else:

scraper/src/config/selectors_parser.py

@@ -1,3 +1,4 @@
+import builtins
 import re
 import copy
 
@@ -55,11 +56,11 @@ def parse(config_start_urls):
                                         start_url['variables'][match]['url'],
                                         start_url['variables'][match]['js'])
                                 else:
-                                    raise Exception(
+                                    raise builtins.Exception(
                                         "Bad arguments for variables." + match + " for url " +
                                         start_url['url'])
                         else:
-                            raise Exception(
+                            raise builtins.Exception(
                                 "Missing " + match + " in variables" + " for url " +
                                 start_url['url'])