Update from 45b7b33

Author: pelson

javascript/scss/app.scss

@@ -19,7 +19,7 @@ $container-max-widths: (
    color: white;
 }
 
-.nav-link:hover {
+.nav-link:hover, .nav-link:focus {
    color: white;
 }
 
@@ -145,3 +145,8 @@ img {
    background: $base-color2 !important;
    color: white !important;
 }
+
+
+.m-1em {
+   margin: 1em;
+}

pyproject.toml

@@ -6,14 +6,68 @@
 # or submit itself to any jurisdiction.
 
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]
+requires = ["setuptools>=61", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "simple-repository-browser"
+dynamic = ["version"]
+description = "A web interface to browse and search packages in any simple package repository (PEP-503), inspired by PyPI / warehouse"
+requires-python = ">=3.11"
+classifiers = [
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+]
+authors = [
+    {name = "Phil Elson"},
+    {name = "Francesco Iannaccone"},
+    {name = "Ivan Sinkarenko"},
+]
+dependencies = [
+  "httpx",
+  "aiosqlite",
+  "diskcache",
+  "docutils",
+  "fastapi",
+  "importlib_metadata>=6.0",
+  "jinja2",
+  "markdown",
+  "markupsafe",
+  "packaging",
+  "parsley",
+  "pkginfo",
+  "readme-renderer[md]",
+  "simple-repository>=0.6",
+  "uvicorn",
+]
+readme = "README.md"
+
+[project.urls]
+Homepage = "https://github.com/simple-repository/simple-repository-browser"
+
+[project.optional-dependencies]
+test = [
+  "pytest",
+]
+dev = [
+  "simple-repository-browser[test]",
+]
+
+[project.scripts]
+simple-repository-browser = "simple_repository_browser.__main__:main"
+
 
 [tool.setuptools_scm]
-write_to = "simple_repository_browser/_version.py"
+version_file = "simple_repository_browser/_version.py"
 
 [[tool.mypy.overrides]]
 module = [
     "diskcache",
     "parsley",
 ]
 ignore_missing_imports = true
+
+[tool.setuptools.packages.find]
+include = ["simple_repository_browser", "simple_repository_browser.*"]
+namespaces = false

setup.py

@@ -26,7 +26,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
     # parsed arguments.
     parser.set_defaults(handler=handler)
 
-    parser.add_argument("index_url", type=str, nargs='?', default='https://pypi.org/simple/')
+    parser.add_argument("repository_url", type=str, nargs='?', default='https://pypi.org/simple/')
     parser.add_argument("--host", default="0.0.0.0")
     parser.add_argument("--port", type=int, default=8080)
     parser.add_argument("--cache-dir", type=str, default=Path(os.environ.get('XDG_CACHE_DIR', Path.home() / '.cache')) / 'simple-repository-browser')
@@ -37,7 +37,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
 
 def handler(args: typing.Any) -> None:
     app = AppBuilder(
-        index_url=args.index_url,
+        repository_url=args.repository_url,
         cache_dir=Path(args.cache_dir),
         template_paths=[
             args.templates_dir,
@@ -69,5 +69,7 @@ def main():
 
 
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.WARNING)
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger('httpcore').setLevel(logging.WARNING)
+    logging.getLogger('httpx').setLevel(logging.WARNING)
     main()

simple_repository_browser/__main__.py

@@ -9,13 +9,16 @@
 import sqlite3
 import typing
 from pathlib import Path
+from urllib.parse import urlparse
 
 import aiosqlite
 import diskcache
 import fastapi
+import fastapi.responses
 import httpx
 from simple_repository import SimpleRepository
 from simple_repository.components.http import HttpRepository
+from simple_repository.components.local import LocalRepository
 
 from . import controller, crawler, errors, fetch_projects, model, view
 from .metadata_injector import MetadataInjector
@@ -25,15 +28,15 @@ class AppBuilder:
     def __init__(
         self,
         url_prefix: str,
-        index_url: str,
+        repository_url: str,
         cache_dir: Path,
         template_paths: typing.Sequence[Path],
         static_files_path: Path,
         crawl_popular_projects: bool,
         browser_version: str,
     ) -> None:
         self.url_prefix = url_prefix
-        self.index_url = index_url
+        self.repository_url = repository_url
         self.cache_dir = cache_dir
         self.template_paths = template_paths
         self.static_files_path = static_files_path
@@ -55,7 +58,7 @@ def create_app(self) -> fastapi.FastAPI:
 
         async def lifespan(app: fastapi.FastAPI):
             async with (
-                httpx.AsyncClient() as http_client,
+                httpx.AsyncClient(timeout=30) as http_client,
                 aiosqlite.connect(self.db_path, timeout=5) as db,
             ):
                 _controller = self.create_controller(
@@ -67,6 +70,15 @@ async def lifespan(app: fastapi.FastAPI):
                 )
                 router = _controller.create_router(self.static_files_path)
                 app.mount(self.url_prefix or "/", router)
+
+                if self.url_prefix:
+                    # If somebody visits the root URL, and that isn't index (because we are
+                    # using a prefix) just redirect them to the index page. This is super
+                    # convenient for development purposes.
+                    @app.get("/")
+                    async def redirect_to_index():
+                        return fastapi.responses.RedirectResponse(url=app.url_path_for('index'))
+
                 yield
 
         app = fastapi.FastAPI(
@@ -84,7 +96,7 @@ async def catch_exceptions_middleware(request: fastapi.Request, call_next):
                 status_code = 500
                 detail = f"Internal server error ({err})"
                 # raise
-                logging.error(err)
+                logging.exception(err)
             content = _view.error_page(
                 request=request,
                 context=model.ErrorModel(detail=detail),
@@ -110,13 +122,18 @@ def create_crawler(self, http_client: httpx.AsyncClient, source: SimpleRepositor
             cache=self.cache,
         )
 
+    def _repo_from_url(self, url: str, http_client: httpx.AsyncClient) -> SimpleRepository:
+        if urlparse(url).scheme in ("http", "https"):
+            return HttpRepository(
+                url=url,
+                http_client=http_client,
+            )
+        else:
+            return LocalRepository(Path(url))
+
     def create_model(self, http_client: httpx.AsyncClient, database: aiosqlite.Connection) -> model.Model:
         source = MetadataInjector(
-            HttpRepository(
-                url=self.index_url,
-                http_client=http_client,
-            ),
-            database=database,
+            self._repo_from_url(self.repository_url, http_client=http_client),
             http_client=http_client,
         )
         return model.Model(

simple_repository_browser/_app.py

@@ -14,7 +14,7 @@
 
 import diskcache
 import httpx
-from packaging.requirements import InvalidRequirement
+from packaging.requirements import InvalidRequirement, Requirement
 from packaging.utils import canonicalize_name
 from packaging.version import Version
 from simple_repository import SimpleRepository, model
@@ -55,19 +55,22 @@ async def crawl_recursively(
         normalized_project_names_to_crawl: typing.Set[str],
     ) -> None:
         """
-        Crawl the matadata of the packages in
-        normalized_project_names_to_crawl and
+        Crawl the metadata of the packages in normalized_project_names_to_crawl and
         of their dependencies.
         """
         seen: set = set()
         packages_for_reindexing = set(normalized_project_names_to_crawl)
         while packages_for_reindexing - seen:
             remaining_packages = packages_for_reindexing - seen
             pkg_name = remaining_packages.pop()
-            logging.info(
+            logging.debug(
                 f"Index iteration loop. Looking at {pkg_name}, with {len(remaining_packages)} remaining ({len(seen)} having been completed)",
             )
             seen.add(pkg_name)
+            if len(seen) % 100 == 0:
+                logging.info(
+                    f"Index iteration batch of 100 complete. {len(seen)} completed, {len(remaining_packages)} remaining",
+                )
             try:
                 prj = await self._source.get_project_page(pkg_name)
             except PackageNotFoundError:
@@ -97,8 +100,9 @@ async def crawl_recursively(
                 continue
 
             for dist in pkg_info.requires_dist:
-                dep_name = dist.name
-                packages_for_reindexing.add(canonicalize_name(dep_name))
+                if isinstance(dist, Requirement):
+                    dep_name = dist.name
+                    packages_for_reindexing.add(canonicalize_name(dep_name))
 
             # Don't DOS the service, we aren't in a rush here.
             await asyncio.sleep(0.01)
@@ -109,7 +113,7 @@ async def refetch_hook(self) -> None:
         # We periodically want to refresh the project database to make sure we are up-to-date.
         await fetch_projects.fully_populate_db(
             connection=self._projects_db,
-            index=self._source,
+            repository=self._source,
         )
         packages_w_dist_info = set()
         for cache_type, name, version in self._cache:
@@ -118,7 +122,7 @@ async def refetch_hook(self) -> None:
 
         popular_projects = []
         if self._crawl_popular_projects:
-            # Add the top 100 packages (and their dependencies) to the index
+            # Add the top 100 packages (and their dependencies) to the repository
             URL = 'https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json'
             try:
                 resp = await self._http_client.get(URL)
@@ -129,10 +133,12 @@ async def refetch_hook(self) -> None:
                 logging.warning(f'Problem fetching popular projects ({err})')
                 pass
 
-        await self.crawl_recursively(packages_w_dist_info | set(popular_projects))
+        projects_to_crawl = packages_w_dist_info | set(popular_projects)
+        logging.info(f'About to start crawling {len(projects_to_crawl)} projects (and their transient dependencies)')
+        await self.crawl_recursively(projects_to_crawl)
 
     async def run_reindex_periodically(self) -> None:
-        logging.info("Starting the reindexing loop")
+        logging.debug("Starting the reindexing loop")
         while True:
             try:
                 await self.refetch_hook()