Skip to content

Use multiprocessing when parsing packages. #170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Use multiprocessing when parsing packages. #170

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a65b88f
Use multiprocessing when parsing packages.
mikepurvis May 10, 2017
9a0641a
Review feedback.
mikepurvis May 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/catkin_pkg/packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
Library to find packages in the filesystem.
"""

import multiprocessing
import os
from .package import parse_package, PACKAGE_MANIFEST_FILENAME

Expand Down Expand Up @@ -93,6 +94,17 @@ def find_packages(basepath, exclude_paths=None, exclude_subspaces=False, warning
return packages


class _PackageParser(object):
def __init__(self, basepath, capture_warnings):
self.basepath = basepath
self.capture_warnings = capture_warnings

def __call__(self, path):
warnings = [] if self.capture_warnings else None
parsed_package = parse_package(os.path.join(self.basepath, path), warnings=warnings)
return (path, parsed_package), warnings


def find_packages_allowing_duplicates(basepath, exclude_paths=None, exclude_subspaces=False, warnings=None):
"""
Crawls the filesystem to find package manifest files and parses them.
Expand All @@ -104,11 +116,12 @@ def find_packages_allowing_duplicates(basepath, exclude_paths=None, exclude_subs
:param warnings: Print warnings if None or return them in the given list
:returns: A dict mapping relative paths to ``Package`` objects ``dict``
"""
packages = {}
package_paths = find_package_paths(basepath, exclude_paths=exclude_paths, exclude_subspaces=exclude_subspaces)
for path in package_paths:
packages[path] = parse_package(os.path.join(basepath, path), warnings=warnings)
return packages
parser = _PackageParser(basepath, warnings is not None)
path_parsed_packages, warnings_lists = zip(*multiprocessing.Pool().map(parser, package_paths))
if parser.capture_warnings:
map(warnings.extend, warnings_lists)
return dict(path_parsed_packages)


def verify_equal_package_versions(packages):
Expand Down