Changed the logic of fetching leader's data

Author: samyak003

backend/apps/owasp/management/commands/owasp_scrape_chapters.py

@@ -63,11 +63,10 @@ def handle(self, *args, **options):
                 else:
                     logger.info("Skipped related URL %s", verified_url)
 
-                chapter.leaders_raw = scraper.get_leaders()
-
+            repository = chapter.owasp_repository
+            chapter.leaders_raw = scraper.get_leaders(repository)
             chapter.invalid_urls = sorted(invalid_urls)
             chapter.related_urls = sorted(related_urls)
-
             chapters.append(chapter)
 
             time.sleep(0.5)

backend/apps/owasp/management/commands/owasp_scrape_committees.py

@@ -63,8 +63,8 @@ def handle(self, *args, **options):
                 else:
                     logger.info("Skipped related URL %s", verified_url)
 
-                committee.leaders_raw = scraper.get_leaders()
-
+            repository = committee.owasp_repository
+            committee.leaders_raw = scraper.get_leaders(repository)
             committee.invalid_urls = sorted(invalid_urls)
             committee.related_urls = sorted(related_urls)
 

backend/apps/owasp/management/commands/owasp_scrape_projects.py

@@ -75,11 +75,10 @@ def handle(self, *args, **options):
                 else:
                     logger.info("Skipped related URL %s", verified_url)
 
-                project.leaders_raw = scraper.get_leaders()
-
+            repository = project.owasp_repository
+            project.leaders_raw = scraper.get_leaders(repository)
             project.invalid_urls = sorted(invalid_urls)
             project.related_urls = sorted(related_urls)
-
             projects.append(project)
 
             time.sleep(0.5)

backend/apps/owasp/scraper.py

@@ -1,13 +1,16 @@
 """OWASP scraper."""
 
 import logging
+import re
 from urllib.parse import urlparse
 
 import requests
 from lxml import etree, html
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from apps.github.utils import get_repository_file_content
+
 logger = logging.getLogger(__name__)
 
 MAX_RETRIES = 3
@@ -55,15 +58,24 @@ def get_urls(self, domain=None):
             else self.page_tree.xpath("//div[@class='sidebar']//a/@href")
         )
 
-    def get_leaders(self):
-        """Get leaders."""
-        leaders_header = self.page_tree.xpath("//div[@class='sidebar']//*[@id='leaders']")
-        if leaders_header:
-            leaders_ul = leaders_header[0].getnext()
-            if leaders_ul is not None and leaders_ul.tag == "ul":
-                return sorted(name.strip() for name in leaders_ul.xpath(".//li/a/text()"))
-
-        return []
+    def get_leaders(self, repository):
+        """Get leaders from leaders.md file on GitHub."""
+        content = get_repository_file_content(
+            f"https://raw.githubusercontent.com/OWASP/{repository.key}/{repository.default_branch}/leaders.md"
+        )
+        leaders = []
+        try:
+            lines = content.split("\n")
+            logger.debug("Content: %s", content)
+            for line in lines:
+                logger.debug("Processing line: %s", line)
+                match = re.findall(r"\* \[([^\]]+)\]", line)
+                leaders.extend(match)
+        except AttributeError:
+            logger.exception(
+                "Unable to parse leaders.md content", extra={"repository": repository.name}
+            )
+        return leaders
 
     def verify_url(self, url):
         """Verify URL."""

backend/tests/owasp/scraper_test.py

@@ -113,10 +113,13 @@ def test_get_leaders_no_leaders(self, mock_session):
         mock_response = Mock()
         mock_response.content = invalid_html
         mock_session.get.return_value = mock_response
-
         scraper = OwaspScraper("https://test.org")
+        mock_repository = Mock()
+        mock_repository.key = "test-repo"
+        mock_repository.default_branch = "main"
+        mock_repository.name = "Test Repository"
 
-        assert scraper.get_leaders() == []
+        assert scraper.get_leaders(mock_repository) == []
 
     def test_verify_url_invalid_url(self, mock_session):
         response = Mock()