diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c4a51c35 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Exclude everything by default +* + +# Whitelist specific files +!src/**/* +!tests/**/* +!setup.py +!requirements.txt +!pytest.ini +!csbot.*.cfg + +# Exclude Python temp files +**/__pycache__ +**/*.pyc \ No newline at end of file diff --git a/.gitignore b/.gitignore index bbc75b2d..ed6ba218 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ cabal-dev *.egg *.egg-info dist -build +build/ eggs parts bin @@ -58,3 +58,9 @@ htmlcov # Linters .ropeproject/ + +# Project-specific files +csbot.cfg +.env +deploy.env +mongodb-data/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 03cde24e..16c31f2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,17 @@ sudo: false dist: xenial language: python -python: - - 3.6 - - 3.7 - install: - - pip install -r requirements.txt + - pip install tox -script: - - pytest -v --cov +matrix: + include: + - python: '3.6' + env: TOXENV=py36-coveralls + - python: '3.7' + env: TOXENV=py37-coveralls -after_success: - - coveralls +script: tox cache: directories: @@ -24,12 +23,3 @@ notifications: - irc.freenode.org#cs-york-dev skip_join: true use_notice: true - -deploy: - provider: heroku - api_key: - secure: jHzS/L/cN/6gCNJrmVCVDb0V4+Zc1b/PnTYcVfoaAw7/USIb2ZQbU6uwPCpGZ8EL/dQlgOCwJY1UYzowm5d6xvXw+9+iHOIBAAgPe0VEmJ2GMPd1/n8cl5CiJ+LF3NXyBml/F4BL/2wm+kZUxINeZfJaim2OAd9g8PfgpHUey5A= - app: csyorkbot - on: - repo: HackSoc/csbot - python: '3.6' diff --git a/Dockerfile b/Dockerfile index 81f59578..ed86684f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,16 @@ -FROM ubuntu:18.04 +FROM python:3.7 -# From python:3.5 docker image, set locale -ENV LANG C.UTF-8 +ARG UID=9000 +ARG GID=9000 -VOLUME /app -WORKDIR /app - -# Update base OS -RUN apt-get -y update && apt-get -y upgrade -# Install Python 3(.4) -RUN apt-get -y install python3 python3-dev python-virtualenv -# Install dependencies for Python libs -RUN apt-get -y install libxml2-dev libxslt1-dev zlib1g-dev +RUN groupadd -g $GID app \ + && useradd -u $UID -g $GID --no-create-home app -# Copy needed files to build docker image -ADD requirements.txt docker-entrypoint.sh ./ - -# Create virtualenv -RUN virtualenv -p python3 /venv -# Populate virtualenv -RUN ./docker-entrypoint.sh pip install --upgrade pip -RUN ./docker-entrypoint.sh pip install -r requirements.txt +COPY --chown=app:app . /app +WORKDIR /app +RUN pip install -r requirements.txt -ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["./run_csbot.py", "csbot.cfg"] +ARG SOURCE_COMMIT +ENV SOURCE_COMMIT $SOURCE_COMMIT +USER app:app +CMD ["csbot", "csbot.cfg"] diff --git a/README.rst b/README.rst index d257d3e3..fd77afdc 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ and running [1]_:: $ python3 -m venv venv3 $ source venv3/bin/activate $ pip install -r requirements.txt - $ ./run_csbot.py --help + $ csbot --help Look at ``csbot.deploy.cfg`` for an example of a bot configuration. @@ -29,6 +29,22 @@ Docker containers (a MongoDB instance and the bot):: $ docker-compose up +This will use the `published image`_. To build locally:: + + $ docker build -t alanbriolat/csbot:latest . + +Environment variables to expose to the bot, e.g. for sensitive configuration +values, should be defined in ``deploy.env``. Environment variables used in +``docker-compose.yml`` should be defined in ``.env``: + +========================== ================== =========== +Variable Default Description +========================== ================== =========== +``CSBOT_CONFIG_LOCAL`` ``./csbot.cfg`` Path to config file in host filesystem to mount at ``/app/csbot.cfg`` +``CSBOT_CONFIG`` ``csbot.cfg`` Path to config file in container, relative to ``/app`` +``CSBOT_WATCHTOWER`` ``false`` Set to ``true`` to use Watchtower_ to auto-update when published container is updated +========================== ================== =========== + Backup MongoDB once services are running:: $ docker-compose exec -T mongodb mongodump --archive --gzip --quiet > foo.mongodump.gz @@ -73,3 +89,5 @@ We're also using Travis-CI for continuous integration and continuous deployment. .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _lxml: http://lxml.de/ .. _Docker Compose: https://docs.docker.com/compose/ +.. _published image: https://hub.docker.com/r/alanbriolat/csbot +.. _Watchtower: https://containrrr.github.io/watchtower/ \ No newline at end of file diff --git a/csbot.deploy.cfg b/csbot.deploy.cfg index 59115288..644dc49b 100644 --- a/csbot.deploy.cfg +++ b/csbot.deploy.cfg @@ -1,8 +1,10 @@ [@bot] +ircv3 = true nickname = Mathison auth_method = sasl_plain channels = #cs-york #cs-york-dev #compsoc-uk #hacksoc #hacksoc-bottest plugins = logger linkinfo hoogle imgur csyork usertrack auth topic helix calc mongodb termdates whois xkcd youtube last webserver webhook github +client_ping = 60 [linkinfo] scan_limit = 2 @@ -47,7 +49,7 @@ end = [webserver] host = 0.0.0.0 -port = 80 +port = 8000 [github] # Re-usable format strings diff --git a/csbot/test/test_events.py b/csbot/test/test_events.py deleted file mode 100644 index 44a90609..00000000 --- a/csbot/test/test_events.py +++ /dev/null @@ -1,363 +0,0 @@ -import unittest -from unittest import mock -import datetime -import collections.abc -import asyncio - -import pytest - -import csbot.events - - -class TestImmediateEventRunner(unittest.TestCase): - def setUp(self): - self.runner = csbot.events.ImmediateEventRunner(self.handle_event) - self.handled_events = [] - - def tearDown(self): - self.runner = None - self.handled_events = None - - def handle_event(self, event): - """Record objects passed through the event handler in order. If they - are callable, call them.""" - self.handled_events.append(event) - if isinstance(event, collections.abc.Callable): - event() - - def test_values(self): - """Check that basic values are passed through the event queue - unmolested.""" - # Test that things actually get through - self.runner.post_event('foo') - self.assertEqual(self.handled_events, ['foo']) - # The event runner doesn't care what it's passing through - for x in ['bar', 1.3, None, object]: - self.runner.post_event(x) - self.assertIs(self.handled_events[-1], x) - - def test_event_chain(self): - """Check that chains of events get handled.""" - def f1(): - self.runner.post_event(f2) - - def f2(): - self.runner.post_event(f3) - - def f3(): - pass - - self.runner.post_event(f1) - self.assertEqual(self.handled_events, [f1, f2, f3]) - - def test_event_tree(self): - """Check that trees of events are handled breadth-first.""" - def f1(): - self.runner.post_event(f2) - self.runner.post_event(f3) - - def f2(): - self.runner.post_event(f4) - - def f3(): - self.runner.post_event(f5) - self.runner.post_event(f6) - - def f4(): - self.runner.post_event(f3) - - def f5(): - pass - - def f6(): - pass - - self.runner.post_event(f1) - self.assertEqual(self.handled_events, - [f1, f2, f3, f4, f5, f6, f3, f5, f6]) - - def test_exception_recovery(self): - """Check that exceptions propagate out of the event runner but don't - leave it broken. - - (In an early version of ImmediateEventRunner, an exception would leave - the runner's queue non-empty and new root events would accumulate - instead of being processed.) - """ - def f1(): - self.runner.post_event(f2) - raise Exception() - - def f2(): - pass - - def f3(): - self.runner.post_event(f4) - - def f4(): - pass - - self.assertRaises(Exception, self.runner.post_event, f1) - self.assertEqual(self.handled_events, [f1]) - self.runner.post_event(f3) - self.assertEqual(self.handled_events, [f1, f3, f4]) - - -@pytest.fixture -def async_runner(event_loop): - def handle_event(event): - if asyncio.iscoroutinefunction(event): - return [event()] - else: - return [] - - obj = mock.Mock() - obj.handle_event = mock.Mock(wraps=handle_event) - obj.runner = csbot.events.AsyncEventRunner(obj.handle_event, event_loop) - obj.exception_handler = mock.Mock(wraps=event_loop.get_exception_handler()) - event_loop.set_exception_handler(obj.exception_handler) - - return obj - - -class TestAsyncEventRunner: - @pytest.mark.asyncio - def test_values(self, async_runner): - """Check that basic values are passed through the event queue - unmolested.""" - # Test that things actually get through - yield from async_runner.runner.post_event('foo') - assert async_runner.handle_event.call_args_list == [mock.call('foo')] - # The event runner doesn't care what it's passing through - for x in ['bar', 1.3, None, object]: - yield from async_runner.runner.post_event(x) - assert async_runner.handle_event.call_args[0][0] is x - - @pytest.mark.asyncio - def test_event_chain(self, async_runner): - """Check that chains of events get handled.""" - @asyncio.coroutine - def f1(): - async_runner.runner.post_event(f2) - - @asyncio.coroutine - def f2(): - async_runner.runner.post_event(f3) - - @asyncio.coroutine - def f3(): - pass - - yield from async_runner.runner.post_event(f1) - assert async_runner.handle_event.call_count == 3 - async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2), mock.call(f3)]) - - @pytest.mark.asyncio(allow_unhandled_exception=True) - def test_exception_recovery(self, async_runner): - """Check that exceptions are handled but don't block other tasks or - leave the runner in a broken state. - """ - @asyncio.coroutine - def f1(): - async_runner.runner.post_event(f2) - raise Exception() - - @asyncio.coroutine - def f2(): - pass - - @asyncio.coroutine - def f3(): - async_runner.runner.post_event(f4) - - @asyncio.coroutine - def f4(): - pass - - assert async_runner.exception_handler.call_count == 0 - yield from async_runner.runner.post_event(f1) - assert async_runner.exception_handler.call_count == 1 - async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2)]) - #self.assertEqual(set(self.handled_events), {f1, f2}) - yield from async_runner.runner.post_event(f3) - assert async_runner.exception_handler.call_count == 1 - async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2), mock.call(f3), mock.call(f4)]) - #self.assertEqual(set(self.handled_events), {f1, f2, f3, f4}) - - -class TestEvent(unittest.TestCase): - class DummyBot(object): - pass - - def _assert_events_equal(self, e1, e2, bot=True, - event_type=True, datetime=True, data=True): - """Test helper for comparing two events. ``=False`` disables - checking that property of the events.""" - if bot: - self.assertIs(e1.bot, e2.bot) - if event_type: - self.assertEqual(e1.event_type, e2.event_type) - if datetime: - self.assertEqual(e1.datetime, e2.datetime) - if data: - for k in list(e1.keys()) + list(e2.keys()): - self.assertEqual(e1[k], e2[k]) - - def test_create(self): - # Test data - data = {'a': 1, 'b': 2, 'c': None} - dt = datetime.datetime.now() - bot = self.DummyBot() - - # Create the event - e = csbot.events.Event(bot, 'event.type', data) - # Check that the event's datetime can be reasonably considered "now" - self.assertTrue(dt <= e.datetime) - self.assertTrue(abs(e.datetime - dt) < datetime.timedelta(seconds=1)) - # Check that the bot, event type and data made it through - self.assertIs(e.bot, bot) - self.assertEqual(e.event_type, 'event.type') - for k, v in data.items(): - self.assertEqual(e[k], v) - - def test_extend(self): - # Test data - data1 = {'a': 1, 'b': 2, 'c': None} - data2 = {'c': 'foo', 'd': 'bar'} - et1 = 'event.type' - et2 = 'other.event' - bot = self.DummyBot() - - # Create an event - e1 = csbot.events.Event(bot, et1, data1) - - # Unchanged event - e2 = csbot.events.Event.extend(e1) - self._assert_events_equal(e1, e2) - - # Change event type only - e3 = csbot.events.Event.extend(e2, et2) - # Check the event type was changed - self.assertEqual(e3.event_type, et2) - # Check that everything else stayed the same - self._assert_events_equal(e1, e3, event_type=False) - - # Change the event type and data - e4 = csbot.events.Event.extend(e1, et2, data2) - # Check the event type was changed - self.assertEqual(e4.event_type, et2) - # Check the data was updated - for k in data1: - if k not in data2: - self.assertEqual(e4[k], data1[k]) - for k in data2: - self.assertEqual(e4[k], data2[k]) - # Check that everything else stayed the same - self._assert_events_equal(e1, e4, event_type=False, data=False) - - -class TestCommandEvent(unittest.TestCase): - def setUp(self): - self.nick = 'csbot' - - def _check_valid_command(self, message, prefix, command, data): - """Test helper for checking the result of parsing a command from a - message.""" - e = csbot.events.Event(None, 'test.event', {'message': message}) - c = csbot.events.CommandEvent.parse_command(e, prefix, self.nick) - self.assertEqual(c['command'], command) - self.assertEqual(c['data'], data) - return c - - def _check_invalid_command(self, message, prefix): - """Test helper for verifying that an invalid command is not - interpreted as a valid command.""" - e = csbot.events.Event(None, 'test.event', {'message': message}) - c = csbot.events.CommandEvent.parse_command(e, prefix, self.nick) - self.assertIs(c, None) - return c - - def test_parse_command(self): - # --> Test variations on command and data text with no prefix involvement - # Just a command - self._check_valid_command('testcommand', '', - 'testcommand', '') - # Command and data - self._check_valid_command('test command data', '', - 'test', 'command data') - # Leading/trailing spaces are ignored - self._check_valid_command(' test command', '', 'test', 'command') - self._check_valid_command('test command ', '', 'test', 'command') - self._check_valid_command(' test command ', '', 'test', 'command') - # Non-alphanumeric commands - self._check_valid_command('!#?$ you !', '', '!#?$', 'you !') - - # --> Test what happens with a command prefix - # Not a command - self._check_invalid_command('just somebody talking', '!') - # A simple command - self._check_valid_command('!hello', '!', 'hello', '') - # ... with data - self._check_valid_command('!hello there', '!', 'hello', 'there') - # ... and repeated prefix - self._check_valid_command('!hello !there everybody', '!', - 'hello', '!there everybody') - # Leading spaces - self._check_valid_command(' !hello', '!', 'hello', '') - # Spaces separating the prefix from the command shouldn't trigger it - self._check_invalid_command('! hello', '!') - # The prefix can be part of the command if repeated - self._check_valid_command('!!hello', '!', '!hello', '') - self._check_valid_command('!!', '!', '!', '') - - # --> Test a longer prefix - # As long as it is a prefix of the first "part", should be fine - self._check_valid_command('dosomething now', 'do', 'something', 'now') - # ... but if there's a space in between it's not a command any more - self._check_invalid_command('do something now', 'do') - - # --> Test unicode - # Unicode prefix - self._check_valid_command('\u0CA0test', '\u0CA0', 'test', '') - # Shouldn't match part of a UTF8 multibyte sequence: \u0CA0 = \xC2\xA3 - self._check_invalid_command('\u0CA0test', '\xC2') - # Unicode command - self._check_valid_command('!\u0CA0_\u0CA0', '!', '\u0CA0_\u0CA0', '') - - # Test "conversational", i.e. mentioned by nick - self._check_valid_command('csbot: do something', '!', 'do', 'something') - self._check_valid_command(' csbot, do something ', '!', 'do', 'something') - self._check_valid_command('csbot:do something', '!', 'do', 'something') - self._check_invalid_command('csbot do something', '!') - - def test_arguments(self): - """Test argument grouping/parsing. These tests are pretty much just - testing :func:`csbot.util.parse_arguments`, which should have its own - tests.""" - # No arguments - c = self._check_valid_command('!foo', '!', 'foo', '') - self.assertEqual(c.arguments(), []) - # Some simple arguments - c = self._check_valid_command('!foo bar baz', '!', 'foo', 'bar baz') - self.assertEqual(c.arguments(), ['bar', 'baz']) - # ... with extra spaces - c = self._check_valid_command('!foo bar baz ', '!', - 'foo', 'bar baz') - self.assertEqual(c.arguments(), ['bar', 'baz']) - # Forced argument grouping with quotes - c = self._check_valid_command('!foo "bar baz"', '!', - 'foo', '"bar baz"') - self.assertEqual(c.arguments(), ['bar baz']) - # ... with extra spaces - c = self._check_valid_command('!foo "bar baz " ', '!', - 'foo', '"bar baz "') - self.assertEqual(c.arguments(), ['bar baz ']) - # Escaped quote preserved - c = self._check_valid_command(r'!foo ba\"r', '!', 'foo', r'ba\"r') - self.assertEqual(c.arguments(), ['ba"r']) - # Unmatched quotes break - c = self._check_valid_command('!foo ba"r', '!', 'foo', 'ba"r') - self.assertRaises(ValueError, c.arguments) - # No mangling in the command part - c = self._check_valid_command('!"foo bar', '!', '"foo', 'bar') - c = self._check_valid_command('"foo bar', '"', 'foo', 'bar') diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py deleted file mode 100644 index ac6c5cde..00000000 --- a/csbot/test/test_plugin_linkinfo.py +++ /dev/null @@ -1,194 +0,0 @@ -# coding=utf-8 -from lxml.etree import LIBXML_VERSION -import unittest.mock as mock - -import pytest -import requests - -from csbot.util import simple_http_get - - -#: Test encoding handling; tests are (url, content-type, body, expected_title) -encoding_test_cases = [ - # (These test case are synthetic, to test various encoding scenarios) - - # UTF-8 with Content-Type header encoding only - ( - "http://example.com/utf8-content-type-only", - "text/html; charset=utf-8", - b"EM DASH \xe2\x80\x94 —", - 'EM DASH \u2014 \u2014' - ), - # UTF-8 with meta http-equiv encoding only - ( - "http://example.com/utf8-meta-http-equiv-only", - "text/html", - (b'' - b'EM DASH \xe2\x80\x94 —'), - 'EM DASH \u2014 \u2014' - ), - # UTF-8 with XML encoding declaration only - ( - "http://example.com/utf8-xml-encoding-only", - "text/html", - (b'' - b'EM DASH \xe2\x80\x94 —'), - 'EM DASH \u2014 \u2014' - ), - - # (The following are real test cases the bot has barfed on in the past) - - # Content-Type encoding, XML encoding declaration *and* http-equiv are all - # present (but no UTF-8 in title). If we give lxml a decoded string with - # the XML encoding declaration it complains. - ( - "http://www.w3.org/TR/REC-xml/", - "text/html; charset=utf-8", - b""" - - - - - - Extensible Markup Language (XML) 1.0 (Fifth Edition) - - - - - - - """, - 'Extensible Markup Language (XML) 1.0 (Fifth Edition)' - ), - # No Content-Type encoding, but has http-equiv encoding. Has a mix of - # UTF-8 literal em-dash and HTML entity em-dash - both should be output as - # unicode em-dash. - ( - "http://docs.python.org/2/library/logging.html", - "text/html", - b""" - - - - - 15.7. logging \xe2\x80\x94 Logging facility for Python — Python v2.7.3 documentation - - - - - - - """, - '15.7. logging \u2014 Logging facility for Python \u2014 Python v2.7.3 documentation' - ), - ( - "http://example.com/invalid-charset", - "text/html; charset=utf-flibble", - b'Flibble', - 'Flibble' - ), -] - -# Add HTML5 test-cases if libxml2 is new enough ( encoding -# detection was added in 2.8.0) -if LIBXML_VERSION >= (2, 8, 0): - encoding_test_cases += [ - # UTF-8 with meta charset encoding only - ( - "http://example.com/utf8-meta-charset-only", - "text/html", - (b'' - b'EM DASH \xe2\x80\x94 —'), - 'EM DASH \u2014 \u2014' - ), - ] - - -error_test_cases = [ - ( - "http://example.com/empty-title-tag", - "text/html", - b'', - ), - ( - "http://example.com/whitespace-title-tag", - "text/html", - b' ', - ), - ( - "http://example.com/no-root-element", - "text/html", - b'=6.2,<7.0 -straight.plugin==1.4.0-post-1 -pymongo>=3.6.0 -requests>=2.9.1,<3.0.0 -lxml>=2.3.5 -google-api-python-client>=1.4.1,<2.0.0 -oauth2client>=3,<4 # google-api-python-client dropped dep in a minor release, generates ImportError warnings -imgurpython>=1.1.6,<2.0.0 -isodate>=0.5.1 -aiohttp>=3.5.1,<4.0 -rollbar - # Requirements for unit testing pytest==4.2.0 pytest-asyncio==0.10.0 pytest-aiohttp==0.3.0 -pytest-cov==2.6.1 +#aioresponses==0.6.0 +git+https://github.com/alanbriolat/aioresponses.git@callback-coroutines#egg=aioresponses +pytest-cov asynctest==0.12.2 aiofastforward==0.0.17 responses -python-coveralls>=2.6.0,<2.7.0 mongomock # Requirements for documentation # (commented out to save build time) #sphinx + +-e . diff --git a/run_csbot.py b/run_csbot.py deleted file mode 100755 index 1aabcd2d..00000000 --- a/run_csbot.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python -from csbot import main -main(auto_envvar_prefix='CSBOT') diff --git a/setup.py b/setup.py index 1addecc7..51b78d8c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,33 @@ -from distutils.core import setup -setup(name='csbot', - version='0.1', - packages=['csbot', 'csbot.plugins'], - ) +import setuptools + + +setuptools.setup( + name='csbot', + version='0.3.0', + author='Alan Briolat', + author_email='alan@briol.at', + url='https://github.com/HackSoc/csbot', + packages=['csbot', 'csbot.plugins'], + package_dir={'': 'src'}, + classifiers=[ + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + install_requires=[ + 'click>=6.2,<7.0', + 'straight.plugin==1.4.0-post-1', + 'pymongo>=3.6.0', + 'requests>=2.9.1,<3.0.0', + 'lxml>=2.3.5', + 'aiogoogle>=0.1.13', + 'isodate>=0.5.1', + 'aiohttp>=3.5.1,<4.0', + 'async_generator', + 'rollbar', + ], + entry_points={ + 'console_scripts': [ + 'csbot = csbot:main', + ], + }, +) diff --git a/csbot/__init__.py b/src/csbot/__init__.py similarity index 53% rename from csbot/__init__.py rename to src/csbot/__init__.py index 0e92fbd5..d47568d0 100644 --- a/csbot/__init__.py +++ b/src/csbot/__init__.py @@ -4,12 +4,28 @@ import os import click +import aiohttp import rollbar from .core import Bot -@click.command(context_settings={'help_option_names': ['-h', '--help']}) +__version__ = None +try: + import pkg_resources + __version__ = pkg_resources.get_distribution('csbot').version +except (pkg_resources.DistributionNotFound, ImportError): + pass + + +LOG = logging.getLogger(__name__) + + +@click.command(context_settings={ + 'help_option_names': ['-h', '--help'], + 'auto_envvar_prefix': 'CSBOT', +}) +@click.version_option(version=__version__) @click.option('--debug', '-d', is_flag=True, default=False, help='Turn on debug logging for the bot.') @click.option('--debug-irc', is_flag=True, default=False, @@ -22,12 +38,30 @@ help='Turn on all debug logging.') @click.option('--colour/--no-colour', 'colour_logging', default=None, help='Use colour in logging. [default: automatic]') -@click.option('--rollbar/--no-rollbar', 'use_rollbar', default=False, - help='Enable Rollbar error reporting.') +@click.option('--rollbar-token', default=None, + help='Rollbar access token, enables Rollbar error reporting.') +@click.option('--github-token', default=None, + help='GitHub "personal access token", enables GitHub deployment reporting.') +@click.option('--github-repo', default=None, + help='GitHub repository to report deployments to.') +@click.option('--env-name', default='development', + help='Deployment environment name. [default: development]') @click.argument('config', type=click.File('r')) -def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colour_logging, use_rollbar): +def main(config, + debug, + debug_irc, + debug_events, + debug_asyncio, + debug_all, + colour_logging, + rollbar_token, + github_token, + github_repo, + env_name): """Run an IRC bot from a configuration file. """ + revision = os.environ.get('SOURCE_COMMIT', None) + # Apply "debug all" option if debug_all: debug = debug_irc = debug_events = debug_asyncio = True @@ -73,10 +107,9 @@ def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colou client = Bot(config) client.bot_setup() - # Configure Rollbar for exception reporting - if use_rollbar: - rollbar.init(os.environ['ROLLBAR_ACCESS_TOKEN'], - os.environ.get('ROLLBAR_ENV', 'development')) + # Configure Rollbar for exception reporting, report deployment + if rollbar_token: + rollbar.init(rollbar_token, env_name) def handler(loop, context): exception = context.get('exception') @@ -84,10 +117,20 @@ def handler(loop, context): exc_info = (type(exception), exception, exception.__traceback__) else: exc_info = None - rollbar.report_exc_info(exc_info) + extra_data = { + 'csbot_event': context.get('csbot_event'), + 'csbot_recent_messages': "\n".join(client.recent_messages), + } + rollbar.report_exc_info(exc_info, extra_data=extra_data) loop.default_exception_handler(context) client.loop.set_exception_handler(handler) + if revision: + client.loop.run_until_complete(rollbar_report_deploy(rollbar_token, env_name, revision)) + + if github_token and github_repo and revision: + client.loop.run_until_complete(github_report_deploy(github_token, github_repo, env_name, revision)) + # Run the client def stop(): client.disconnect() @@ -100,6 +143,67 @@ def stop(): client.bot_teardown() +async def rollbar_report_deploy(rollbar_token, env_name, revision): + async with aiohttp.ClientSession() as session: + request = session.post( + 'https://api.rollbar.com/api/1/deploy/', + data={ + 'access_token': rollbar_token, + 'environment': env_name, + 'revision': revision, + }, + ) + async with request as response: + data = await response.json() + if response.status == 200: + LOG.info('Reported deploy to Rollbar: env=%s revision=%s deploy_id=%s', + env_name, revision, data['data']['deploy_id']) + else: + LOG.error('Error reporting deploy to Rollbar: %s', data['message']) + + +async def github_report_deploy(github_token, github_repo, env_name, revision): + headers = { + 'Authorization': f'token {github_token}', + 'Accept': 'application/vnd.github.v3+json', + } + async with aiohttp.ClientSession(headers=headers) as session: + create_request = session.post( + f'https://api.github.com/repos/{github_repo}/deployments', + json={ + 'ref': revision, + 'auto_merge': False, + 'environment': env_name, + 'description': 'Bot running with new version', + }, + ) + async with create_request as create_response: + if create_response.status != 201: + LOG.error('Error reporting deploy to GitHub (create deploy): %s %s\n%s', + create_response.status, create_response.reason, await create_response.text()) + return + + deploy = await create_response.json() + + status_request = session.post( + deploy['statuses_url'], + json={ + 'state': 'success', + + }, + ) + async with status_request as status_response: + if status_response.status != 201: + LOG.error('Error reporting deploy to GitHub (update status): %s %s\n%s', + create_response.status, create_response.reason, await create_response.text()) + return + + status = await status_response.json() + + LOG.info('Reported deploy to GitHub: env=%s revision=%s deploy_id=%s', + env_name, revision, deploy["id"]) + + class PrettyStreamHandler(logging.StreamHandler): """Wrap log messages with severity-dependent ANSI terminal colours. diff --git a/csbot/_rfc.py b/src/csbot/_rfc.py similarity index 100% rename from csbot/_rfc.py rename to src/csbot/_rfc.py diff --git a/csbot/core.py b/src/csbot/core.py similarity index 93% rename from csbot/core.py rename to src/csbot/core.py index e196c1c8..7bab9b15 100644 --- a/csbot/core.py +++ b/src/csbot/core.py @@ -1,8 +1,6 @@ import collections import itertools -import asyncio - import configparser import straight.plugin @@ -10,6 +8,7 @@ from csbot.plugin import build_plugin_dict, PluginManager import csbot.events as events from csbot.events import Event, CommandEvent +from csbot.util import maybe_future_result from .irc import IRCClient, IRCUser @@ -23,6 +22,7 @@ class Bot(SpecialPlugin, IRCClient): #: Default configuration values CONFIG_DEFAULTS = { + 'ircv3': False, 'nickname': 'csyorkbot', 'password': None, 'auth_method': 'pass', @@ -71,6 +71,7 @@ def __init__(self, config=None, loop=None): IRCClient.__init__( self, loop=loop, + ircv3=self.config_getboolean('ircv3'), nick=self.config_get('nickname'), username=self.config_get('username'), host=self.config_get('irc_host'), @@ -82,6 +83,8 @@ def __init__(self, config=None, loop=None): client_ping_interval=int(self.config_get('client_ping')), ) + self._recent_messages = collections.deque(maxlen=10) + # Plumb in reply(...) method if self.config_getboolean('use_notice'): self.reply = self.notice @@ -95,7 +98,7 @@ def __init__(self, config=None, loop=None): self.commands = {} # Event runner - self.events = events.AsyncEventRunner(self._fire_hooks, self.loop) + self.events = events.HybridEventRunner(self._get_hooks, self.loop) # Keeps partial name lists between RPL_NAMREPLY and # RPL_ENDOFNAMES events @@ -111,9 +114,8 @@ def bot_teardown(self): """ self.plugins.teardown() - def _fire_hooks(self, event): - results = self.plugins.fire_hooks(event) - return list(itertools.chain(*results)) + def _get_hooks(self, event): + return itertools.chain(*self.plugins.get_hooks(event.event_type)) def post_event(self, event): return self.events.post_event(event) @@ -161,8 +163,7 @@ def privmsg(self, event): self.post_event(command) @Plugin.hook('core.command') - @asyncio.coroutine - def fire_command(self, event): + async def fire_command(self, event): """Dispatch a command event to its callback. """ # Ignore unknown commands @@ -170,9 +171,7 @@ def fire_command(self, event): return f, _, _ = self.commands[event['command']] - if not asyncio.iscoroutinefunction(f): - f = asyncio.coroutine(f) - yield from f(event) + await maybe_future_result(f(event), log=self.log) @Plugin.command('help', help=('help [command]: show help for command, or ' 'show available commands')) @@ -205,12 +204,14 @@ def emit(self, event): """ self.bot.post_event(event) - def connection_made(self): - super().connection_made() + async def connection_made(self): + await super().connection_made() + if self.config_getboolean('ircv3'): + await self.request_capabilities(enable={'account-notify', 'extended-join'}) self.emit_new('core.raw.connected') - def connection_lost(self, exc): - super().connection_lost(exc) + async def connection_lost(self, exc): + await super().connection_lost(exc) self.emit_new('core.raw.disconnected', {'reason': repr(exc)}) def send_line(self, line): @@ -218,10 +219,15 @@ def send_line(self, line): self.emit_new('core.raw.sent', {'message': line}) def line_received(self, line): + self._recent_messages.append(line) fut = self.emit_new('core.raw.received', {'message': line}) super().line_received(line) return fut + @property + def recent_messages(self): + return list(self._recent_messages) + def on_welcome(self): self.emit_new('core.self.connected') @@ -328,14 +334,6 @@ def on_names(self, channel, names, raw_names): 'raw_names': raw_names, }) - # "IRC Client Capabilities" - - def on_capabilities_available(self, capabilities): - super().on_capabilities_available(capabilities) - for cap in ['account-notify', 'extended-join']: - if cap in capabilities and cap not in self.enabled_capabilities: - self.enable_capability(cap) - # Implement active account discovery via "formatted WHO" def identify(self, target): diff --git a/csbot/events.py b/src/csbot/events.py similarity index 60% rename from csbot/events.py rename to src/csbot/events.py index c7692e0c..7309c6a8 100644 --- a/csbot/events.py +++ b/src/csbot/events.py @@ -4,11 +4,10 @@ import asyncio import logging -from csbot.util import parse_arguments +from csbot.util import parse_arguments, maybe_future LOG = logging.getLogger('csbot.events') -LOG.setLevel(logging.INFO) class ImmediateEventRunner(object): @@ -159,6 +158,152 @@ def _run(self): not_done.add(new_pending) +class HybridEventRunner: + """ + A hybrid synchronous/asynchronous event runner. + + *get_handlers* is called for each event passed to :meth:`post_event`, and + should return an iterable of callables to handle that event, each of which + will be called with the event object. + + Events are processed in the order they are received, with all handlers for + an event being called before the handlers for the next event. If a handler + returns an awaitable, it is added to a set of asynchronous tasks to wait on. + + The future returned by :meth:`post_event` completes only when all events + have been processed and all asynchronous tasks have completed. + + :param get_handlers: Get functions to call for an event + :param loop: asyncio event loop to use (default: use current loop) + """ + def __init__(self, get_handlers, loop=None): + self.get_handlers = get_handlers + self.loop = loop + + self.events = deque() + self.new_events = asyncio.Event(loop=self.loop) + self.futures = set() + self.future = None + + def __enter__(self): + LOG.debug('entering event runner') + + def __exit__(self, exc_type, exc_value, traceback): + LOG.debug('exiting event runner') + self.future = None + + def post_event(self, event): + """Post *event* to be handled soon. + + *event* is added to the queue of events. + + Returns a future which resolves when the handlers of *event* (and all + events generated during those handlers) have completed. + """ + self.events.append(event) + LOG.debug('added event %s, pending=%s', event, len(self.events)) + self.new_events.set() + if not self.future: + self.future = self.loop.create_task(self._run()) + return self.future + + def _run_events(self): + """Run event handlers, accumulating awaitables as futures. + """ + new_futures = set() + while len(self.events) > 0: + LOG.debug('processing events (%s remaining)', len(self.events)) + # Get next event + event = self.events.popleft() + LOG.debug('processing event: %s', event) + # Handle the event + for handler in self.get_handlers(event): + # Attempt to run the handler, but don't break everything if the handler fails + LOG.debug('running handler: %r', handler) + future = self._run_handler(handler, event) + if future: + new_futures.add(future) + self.new_events.clear() + if len(new_futures) > 0: + LOG.debug('got %s new futures', len(new_futures)) + return new_futures + + def _run_handler(self, handler, event): + """Call *handler* with *event* and log any exception. + + If *handler* returns an awaitable, then it is wrapped in a coroutine that will log any + exception from awaiting it. + """ + result = None + try: + result = handler(event) + except Exception as e: + self._handle_exception(exception=e, csbot_event=event) + future = maybe_future( + result, + log=LOG, + loop=self.loop, + ) + if future: + future = asyncio.ensure_future(self._finish_async_handler(future, event), loop=self.loop) + return future + + async def _finish_async_handler(self, future, event): + """Await *future* and log any exception. + """ + try: + await future + except Exception as e: + self._handle_exception(future=future, csbot_event=event) + + async def _run(self): + """Run the event runner loop. + + Process events and await futures until all events and handlers have been + processed. + """ + # Use self as context manager so an escaping exception doesn't break + # the event runner instance permanently (i.e. we clean up the future) + with self: + # Run until no more events or lingering futures + while len(self.events) + len(self.futures) > 0: + # Synchronously run event handler and collect new futures + new_futures = self._run_events() + self.futures |= new_futures + # Don't bother waiting if no futures to wait on + if len(self.futures) == 0: + continue + + # Run until one or more futures complete (or new events are added) + new_events = self.loop.create_task(self.new_events.wait()) + LOG.debug('waiting on %s futures', len(self.futures)) + done, pending = await asyncio.wait(self.futures | {new_events}, + loop=self.loop, + return_when=asyncio.FIRST_COMPLETED) + # Remove done futures from the set of futures being waited on + done_futures = done - {new_events} + LOG.debug('%s of %s futures done', len(done_futures), len(self.futures)) + self.futures -= done_futures + if new_events.done(): + LOG.debug('new events to process') + else: + # If no new events, cancel the waiter, because we'll create a new one next iteration + new_events.cancel() + + def _handle_exception(self, *, message='Unhandled exception in event handler', + exception=None, + future=None, + csbot_event=None): + if exception is None and future is not None: + exception = future.exception() + self.loop.call_exception_handler({ + 'message': message, + 'exception': exception, + 'future': future, + 'csbot_event': csbot_event, + }) + + class Event(dict): """IRC event information. diff --git a/csbot/irc.py b/src/csbot/irc.py similarity index 80% rename from csbot/irc.py rename to src/csbot/irc.py index 764ba6ce..48ff86ae 100644 --- a/csbot/irc.py +++ b/src/csbot/irc.py @@ -6,6 +6,7 @@ import codecs import base64 import types +from typing import Callable, Tuple, Any, Iterable, Awaitable from ._rfc import NUMERIC_REPLIES @@ -190,6 +191,10 @@ def decode(self, input, errors='strict'): return codecs.decode(input, 'cp1252', 'replace') +class IRCClientError(Exception): + pass + + class IRCClient: """Internet Relay Chat client protocol. @@ -228,6 +233,7 @@ class IRCClient: #: Generate a default configuration. Easier to call this and update the #: result than relying on ``dict.copy()``. DEFAULTS = staticmethod(lambda: dict( + ircv3=False, nick='csbot', username=None, host='irc.freenode.net', @@ -256,9 +262,12 @@ def __init__(self, *, loop=None, **kwargs): self.connected.clear() self.disconnected = asyncio.Event(loop=self.loop) self.disconnected.set() + self._last_message_received = self.loop.time() self._client_ping = None self._client_ping_counter = 0 + self._message_waiters = set() + self.nick = self.__config['nick'] self.available_capabilities = set() self.enabled_capabilities = set() @@ -270,9 +279,11 @@ async def run(self, run_once=False): await self.connect() self.connected.set() self.disconnected.clear() - self.connection_made() - await self.read_loop() - self.connection_lost(self.reader.exception()) + # Need to start read_loop() first so that connection_made() can await messages + read_loop_fut = self.loop.create_task(self.read_loop()) + await self.connection_made() + await read_loop_fut + await self.connection_lost(self.reader.exception()) self.connected.clear() self.disconnected.set() if self._exiting: @@ -315,7 +326,7 @@ async def read_loop(self): break self.line_received(self.codec.decode(line[:-2])) - def connection_made(self): + async def connection_made(self): """Callback for successful connection. Register with the IRC server. @@ -328,38 +339,44 @@ def connection_made(self): password = self.__config['password'] auth_method = self.__config['auth_method'] + if self.__config['ircv3']: + # Discover available capabilities + self.send(IRCMessage.create('CAP', ['LS'])) + await self.wait_for_message(lambda m: (m.command == 'CAP' and m.params[1] == 'LS', m)) + if auth_method == 'pass': if password: self.send(IRCMessage.create('PASS', [password])) self.set_nick(nick) self.send(user_msg) elif auth_method == 'sasl_plain': - # Just assume the server is going to understand our attempt at SASL - # authentication... - # TODO: proper stateful capability negotiation at this step - self.enable_capability('sasl') + sasl_enabled = await self.request_capabilities(enable={'sasl'}) self.set_nick(nick) self.send(user_msg) - self.send(IRCMessage.create('AUTHENTICATE', ['PLAIN'])) - # SASL PLAIN authentication message (https://tools.ietf.org/html/rfc4616) - # (assuming authzid = authcid = nick) - sasl_plain = '{}\0{}\0{}'.format(nick, nick, password) - # Well this is awkward... password string encoded to bytes as utf-8, - # base64-encoded to different bytes, converted back to string for - # use in the IRCMessage (which later encodes it as utf-8...) - sasl_plain_b64 = base64.b64encode(sasl_plain.encode('utf-8')).decode('ascii') - self.send(IRCMessage.create('AUTHENTICATE', [sasl_plain_b64])) - self.send(IRCMessage.create('CAP', ['END'])) + if sasl_enabled: + self.send(IRCMessage.create('AUTHENTICATE', ['PLAIN'])) + # SASL PLAIN authentication message (https://tools.ietf.org/html/rfc4616) + # (assuming authzid = authcid = nick) + sasl_plain = '{}\0{}\0{}'.format(nick, nick, password) + # Well this is awkward... password string encoded to bytes as utf-8, + # base64-encoded to different bytes, converted back to string for + # use in the IRCMessage (which later encodes it as utf-8...) + sasl_plain_b64 = base64.b64encode(sasl_plain.encode('utf-8')).decode('ascii') + self.send(IRCMessage.create('AUTHENTICATE', [sasl_plain_b64])) + sasl_success = await self.wait_for_message(lambda m: (m.command in ('903', '904'), m.command == '903')) + if not sasl_success: + LOG.error('SASL authentication failed') + else: + LOG.error('could not enable "sasl" capability, skipping authentication') else: raise ValueError('unknown auth_method: {}'.format(auth_method)) - # Discover available client capabilities, if any, which should get - # enabled in callbacks triggered by the CAP LS response - self.send_line('CAP LS') + if self.__config['ircv3']: + self.send(IRCMessage.create('CAP', ['END'])) self._start_client_pings() - def connection_lost(self, exc): + async def connection_lost(self, exc): """Handle a broken connection by attempting to reconnect. Won't reconnect if the broken connection was deliberate (i.e. @@ -371,12 +388,14 @@ def connection_lost(self, exc): def line_received(self, line): """Callback for received raw IRC message.""" + self._last_message_received = self.loop.time() msg = IRCMessage.parse(line) LOG.debug('>>> %s', msg.pretty) self.message_received(msg) def message_received(self, msg): """Callback for received parsed IRC message.""" + self.process_wait_for_message(msg) self._dispatch_method('irc_' + msg.command_name, msg) def send_line(self, data): @@ -407,33 +426,101 @@ def _stop_client_pings(self): self._client_ping = None async def _send_client_pings(self, interval): + """Send a client ``PING`` if no messages have been received for *interval* seconds.""" self._client_ping_counter = 0 + delay = interval while True: - await asyncio.sleep(interval) - self._client_ping_counter += 1 - self.send_line(f'PING {self._client_ping_counter}') + await asyncio.sleep(delay) + now = self.loop.time() + remaining = self._last_message_received + interval - now + + if remaining <= 0: + # Send the PING + self._client_ping_counter += 1 + self.send_line(f'PING {self._client_ping_counter}') + # Wait for another interval + delay = interval + else: + # Wait until interval has elapsed since last message + delay = remaining - # Specific commands for sending messages - def enable_capability(self, name): - """Enable client capability *name*. + class Waiter: + predicate = None + future = None - Should wait for :meth:`on_capability_enabled` before assuming it is - enabled. + def __init__(self, predicate, future): + self.predicate = predicate + self.future = future + + def wait_for_message(self, predicate: Callable[[IRCMessage], Tuple[bool, Any]]) -> asyncio.Future: + """Wait for a message that matches *predicate*. + + *predicate* should return a `(did_match, result)` tuple, where *did_match* is a boolean + indicating if the message is a match, and *result* is the value to return. + + Returns a future that is resolved with *result* on the first matching message. """ - if name not in self.available_capabilities: - LOG.warning('Enabling client capability "{}" not in response to CAP LS'.format(name)) - self.send_line('CAP REQ :{}'.format(name)) + waiter = self.Waiter(predicate, self.loop.create_future()) + self._message_waiters.add(waiter) + return waiter.future + + def process_wait_for_message(self, msg): + done = set() + for w in self._message_waiters: + if not w.future.done(): + matched, result = False, None + try: + matched, result = w.predicate(msg) + except Exception as e: + w.future.set_exception(e) + if matched: + w.future.set_result(result) + if w.future.done(): + done.add(w) + self._message_waiters.difference_update(done) - def disable_capability(self, name): - """Disable client capability *name*. + # Specific commands for sending messages + + def request_capabilities(self, *, enable: Iterable[str] = None, disable: Iterable[str] = None) -> Awaitable[bool]: + """Request a change to the enabled IRCv3 capabilities. - Should wait for :meth:`on_capability_disabled` befor assuming it is - disabled. + *enable* and *disable* are sets of capability names, with *disable* taking precedence. + + Returns a future which resolves with True if the request is successful, or False otherwise. """ - if name not in self.available_capabilities: - LOG.warning('Disabling client capability "{}" not in response to CAP LS'.format(name)) - self.send_line('CAP REQ :-{}'.format(name)) + if not self.__config['ircv3']: + raise IRCClientError('configured with ircv3=False, cannot use capability negotiation') + + enable_set = set(enable or ()) + disable_set = set(disable or ()) + enable_set.difference_update(disable_set) + unknown = enable_set.union(disable_set).difference(self.available_capabilities) + if unknown: + LOG.warning('attempting to request unknown capabilities: %r', unknown) + + request = ' '.join(sorted(enable_set) + [f'-{c}' for c in sorted(disable_set)]) + if len(request) == 0: + LOG.warning('no capabilities requested, not sending CAP REQ') + fut = self.loop.create_future() + fut.set_result(True) + return fut + else: + message = IRCMessage.create('CAP', ['REQ', request]) + self.send(message) + return self._wait_for_capability_response(request) + + def _wait_for_capability_response(self, request): + def predicate(msg): + if msg.command == 'CAP': + _, subcommand, response = msg.params + response = response.strip() + if subcommand == 'ACK' and response == request: + return True, True + elif subcommand == 'NAK' and response == request: + return True, False + return False, None + return self.wait_for_message(predicate) def set_nick(self, nick): """Ask the server to set our nick.""" diff --git a/csbot/plugin.py b/src/csbot/plugin.py similarity index 96% rename from csbot/plugin.py rename to src/csbot/plugin.py index 43eb7dd3..e52fa63f 100644 --- a/csbot/plugin.py +++ b/src/csbot/plugin.py @@ -2,7 +2,7 @@ from collections import abc import logging import os -import asyncio +from typing import List, Callable def build_plugin_dict(plugins): @@ -268,19 +268,10 @@ class Foo(Plugin): """ return ProvidedByPlugin(other, kwargs) - def fire_hooks(self, event): - """Execute all of this plugin's handlers for *event*. - - All handlers are treated as coroutine functions, and the return value is - a list of all the invoked coroutines. + def get_hooks(self, hook: str) -> List[Callable]: + """Get a list of this plugin's handlers for *hook*. """ - coros = [] - for name in self.plugin_hooks.get(event.event_type, ()): - f = getattr(self, name) - if not asyncio.iscoroutinefunction(f): - f = asyncio.coroutine(f) - coros.append(f(event)) - return coros + return [getattr(self, name) for name in self.plugin_hooks.get(hook, ())] def provide(self, plugin_name, **kwarg): """Provide a value for a :meth:`Plugin.use` usage.""" diff --git a/csbot/plugins/__init__.py b/src/csbot/plugins/__init__.py similarity index 100% rename from csbot/plugins/__init__.py rename to src/csbot/plugins/__init__.py diff --git a/csbot/plugins/auth.py b/src/csbot/plugins/auth.py similarity index 100% rename from csbot/plugins/auth.py rename to src/csbot/plugins/auth.py diff --git a/csbot/plugins/calc.py b/src/csbot/plugins/calc.py similarity index 100% rename from csbot/plugins/calc.py rename to src/csbot/plugins/calc.py diff --git a/csbot/plugins/cron.py b/src/csbot/plugins/cron.py similarity index 100% rename from csbot/plugins/cron.py rename to src/csbot/plugins/cron.py diff --git a/csbot/plugins/csyork.py b/src/csbot/plugins/csyork.py similarity index 100% rename from csbot/plugins/csyork.py rename to src/csbot/plugins/csyork.py diff --git a/csbot/plugins/github.py b/src/csbot/plugins/github.py similarity index 100% rename from csbot/plugins/github.py rename to src/csbot/plugins/github.py diff --git a/csbot/plugins/helix.py b/src/csbot/plugins/helix.py similarity index 100% rename from csbot/plugins/helix.py rename to src/csbot/plugins/helix.py diff --git a/csbot/plugins/hoogle.py b/src/csbot/plugins/hoogle.py similarity index 54% rename from csbot/plugins/hoogle.py rename to src/csbot/plugins/hoogle.py index e0ff7718..e42743b1 100644 --- a/csbot/plugins/hoogle.py +++ b/src/csbot/plugins/hoogle.py @@ -1,8 +1,7 @@ -import requests import urllib.parse from csbot.plugin import Plugin -from csbot.util import simple_http_get +from csbot.util import simple_http_get_async class Hoogle(Plugin): @@ -14,43 +13,45 @@ def setup(self): super(Hoogle, self).setup() @Plugin.command('hoogle') - def search_hoogle(self, e): + async def search_hoogle(self, e): """Search Hoogle with a given string and return the first few (exact number configurable) results. """ query = e['data'] hurl = 'http://www.haskell.org/hoogle/?mode=json&hoogle=' + query - hresp = simple_http_get(hurl) + async with simple_http_get_async(hurl) as hresp: - if hresp.status_code != requests.codes.ok: - self.log.warn('request failed for ' + hurl) - return + if hresp.status != 200: + self.log.warn('request failed for ' + hurl) + return + + # The Hoogle response JSON is of the following format: + # { + # "version": "" + # "results": [ + # { + # "location": "" + # "self": " :: " + # "docs": "" + # }, + # ... + # ] + # } + + maxresults = int(self.config_get('results')) + + json = await hresp.json() - # The Hoogle response JSON is of the following format: - # { - # "version": "" - # "results": [ - # { - # "location": "" - # "self": " :: " - # "docs": "" - # }, - # ... - # ] - # } - - maxresults = int(self.config_get('results')) - - if hresp.json is None: + if json is None: self.log.warn('invalid JSON received from Hoogle') return - if 'parseError' in hresp.json(): - e.reply(hresp.json()['parseError'].replace('\n', ' ')) + if 'parseError' in json: + e.reply(json['parseError'].replace('\n', ' ')) return - allresults = hresp.json()['results'] + allresults = json['results'] totalresults = len(allresults) results = allresults[0:maxresults] niceresults = [] diff --git a/csbot/plugins/imgur.py b/src/csbot/plugins/imgur.py similarity index 51% rename from csbot/plugins/imgur.py rename to src/csbot/plugins/imgur.py index 8f832d55..76e99a69 100644 --- a/csbot/plugins/imgur.py +++ b/src/csbot/plugins/imgur.py @@ -1,11 +1,12 @@ -from imgurpython import ImgurClient -from imgurpython.helpers.error import ImgurClientError - from ..plugin import Plugin -from ..util import pluralize +from ..util import pluralize, simple_http_get_async from .linkinfo import LinkInfoResult +class ImgurError(Exception): + pass + + class Imgur(Plugin): CONFIG_DEFAULTS = { 'client_id': None, @@ -17,17 +18,12 @@ class Imgur(Plugin): 'client_secret': ['IMGUR_CLIENT_SECRET'], } - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.client = ImgurClient(self.config_get('client_id'), - self.config_get('client_secret')) - @Plugin.integrate_with('linkinfo') def integrate_with_linkinfo(self, linkinfo): linkinfo.register_handler(lambda url: url.netloc in ('imgur.com', 'i.imgur.com'), self._linkinfo_handler, exclusive=True) - def _linkinfo_handler(self, url, match): + async def _linkinfo_handler(self, url, match): # Split up endpoint and ID: /, /a/ or /gallery/ kind, _, id = url.path.lstrip('/').rpartition('/') # Strip file extension from direct image links @@ -35,18 +31,18 @@ def _linkinfo_handler(self, url, match): try: if kind == '': - nsfw, title = self._format_image(self.client.get_image(id)) + nsfw, title = self._format_image(await self._get_image(id)) elif kind == 'a': - nsfw, title = self._format_album(self.client.get_album(id), url.fragment) + nsfw, title = self._format_album(await self._get_album(id), url.fragment) elif kind == 'gallery': - data = self.client.gallery_item(id) - if data.is_album: + data = await self._get_gallery_item(id) + if data['is_album']: nsfw, title = self._format_album(data, None) else: nsfw, title = self._format_image(data) else: nsfw, title = False, None - except ImgurClientError as e: + except ImgurError as e: return LinkInfoResult(url, str(e), is_error=True) if title: @@ -56,15 +52,33 @@ def _linkinfo_handler(self, url, match): @staticmethod def _format_image(data): - title = data.title or '' - return data.nsfw or 'nsfw' in title.lower(), title + title = data['title'] or '' + return data['nsfw'] or 'nsfw' in title.lower(), title @staticmethod def _format_album(data, image_id): - title = '{0} ({1})'.format(data.title or 'Untitled album', - pluralize(data.images_count, 'image', 'images')) - images = {i['id']: i for i in data.images} + title = '{0} ({1})'.format(data['title'] or 'Untitled album', + pluralize(data['images_count'], 'image', 'images')) + images = {i['id']: i for i in data['images']} image = images.get(image_id) if image and image['title']: title += ': ' + image['title'] - return data.nsfw or 'nsfw' in title.lower(), title + return data['nsfw'] or 'nsfw' in title.lower(), title + + async def _get(self, url): + headers = {'Authorization': f'Client-ID {self.config_get("client_id")}'} + async with simple_http_get_async(url, headers=headers) as resp: + json = await resp.json() + if json['success']: + return json['data'] + else: + raise ImgurError(json['data']['error']) + + async def _get_image(self, id): + return await self._get(f'https://api.imgur.com/3/image/{id}') + + async def _get_album(self, id): + return await self._get(f'https://api.imgur.com/3/album/{id}') + + async def _get_gallery_item(self, id): + return await self._get(f'https://api.imgur.com/3/gallery/{id}') diff --git a/csbot/plugins/last.py b/src/csbot/plugins/last.py similarity index 100% rename from csbot/plugins/last.py rename to src/csbot/plugins/last.py diff --git a/csbot/plugins/linkinfo.py b/src/csbot/plugins/linkinfo.py similarity index 93% rename from csbot/plugins/linkinfo.py rename to src/csbot/plugins/linkinfo.py index 4e6c2014..7dbca490 100644 --- a/csbot/plugins/linkinfo.py +++ b/src/csbot/plugins/linkinfo.py @@ -5,14 +5,13 @@ from collections import namedtuple import datetime from functools import partial -from contextlib import closing -import requests +import aiohttp import lxml.etree import lxml.html from ..plugin import Plugin -from ..util import simple_http_get, Struct +from ..util import Struct, simple_http_get_async, maybe_future_result LinkInfoHandler = namedtuple('LinkInfoHandler', ['filter', 'handler', 'exclusive']) @@ -103,7 +102,7 @@ def register_exclude(self, filter): self.excludes.append(filter) @Plugin.command('link') - def link_command(self, e): + async def link_command(self, e): """Handle the "link" command. Fetch information about a specified URL, e.g. @@ -123,7 +122,7 @@ def link_command(self, e): url = 'http://' + url # Get info for the URL - result = self.get_link_info(url) + result = await self.get_link_info(url) self._log_if_error(result) # See if it was marked as NSFW in the command text result.nsfw |= 'nsfw' in rest.lower() @@ -131,7 +130,7 @@ def link_command(self, e): e.reply(result.get_message()) @Plugin.hook('core.message.privmsg') - def scan_privmsg(self, e): + async def scan_privmsg(self, e): """Scan the data of PRIVMSG events for URLs and respond with information about them. """ @@ -152,7 +151,7 @@ def scan_privmsg(self, e): break # Get info for the URL - result = self.get_link_info(part) + result = await self.get_link_info(part) self._log_if_error(result) if result.is_error: @@ -168,7 +167,7 @@ def scan_privmsg(self, e): # ... and since we got a useful result, stop processing the message break - def get_link_info(self, original_url): + async def get_link_info(self, original_url): """Get information about a URL. Using the *original_url* string, run the chain of URL handlers and @@ -186,7 +185,7 @@ def get_link_info(self, original_url): for h in self.handlers: match = h.filter(url) if match: - result = h.handler(url, match) + result = await maybe_future_result(h.handler(url, match), log=self.log) if result is not None: # Useful result, return it return result @@ -205,11 +204,11 @@ def get_link_info(self, original_url): # Invoke the default handler if not excluded else: try: - return self.scrape_html_title(url) - except requests.exceptions.ConnectionError: + return await self.scrape_html_title(url) + except aiohttp.ClientConnectionError: return make_error('Connection error') - def scrape_html_title(self, url): + async def scrape_html_title(self, url): """Scrape the ```` tag contents from the HTML page at *url*. Returns a :class:`LinkInfoResult`. @@ -217,11 +216,11 @@ def scrape_html_title(self, url): make_error = partial(LinkInfoResult, url.geturl(), is_error=True) # Let's see what's on the other end... - with closing(simple_http_get(url.geturl(), stream=True)) as r: + async with simple_http_get_async(url.geturl()) as r: # Only bother with 200 OK - if r.status_code != requests.codes.ok: - return make_error('HTTP request failed: {}' - .format(r.status_code)) + if r.status != 200: + return make_error('HTTP request failed: {} {}' + .format(r.status, r.reason)) # Only process HTML-ish responses if 'Content-Type' not in r.headers: return make_error('No Content-Type header') @@ -240,8 +239,8 @@ def scrape_html_title(self, url): # If present, charset attribute in HTTP Content-Type header takes # precedence, but fallback to default if encoding isn't recognised parser = lxml.html.html_parser - if 'charset=' in r.headers['content-type']: - encoding = r.headers['content-type'].rsplit('=', 1)[1] + if r.charset is not None: + encoding = r.charset try: parser = lxml.html.HTMLParser(encoding=encoding) except LookupError: @@ -252,7 +251,7 @@ def scrape_html_title(self, url): # because chunk-encoded responses iterate over chunks rather than # the size we request... chunk = b'' - for next_chunk in r.iter_content(self.config_get('max_response_size')): + async for next_chunk in r.content.iter_chunked(self.config_get('max_response_size')): chunk += next_chunk if len(chunk) >= self.config_get('max_response_size'): break diff --git a/csbot/plugins/logger.py b/src/csbot/plugins/logger.py similarity index 100% rename from csbot/plugins/logger.py rename to src/csbot/plugins/logger.py diff --git a/csbot/plugins/mongodb.py b/src/csbot/plugins/mongodb.py similarity index 100% rename from csbot/plugins/mongodb.py rename to src/csbot/plugins/mongodb.py diff --git a/csbot/plugins/termdates.py b/src/csbot/plugins/termdates.py similarity index 100% rename from csbot/plugins/termdates.py rename to src/csbot/plugins/termdates.py diff --git a/csbot/plugins/topic.py b/src/csbot/plugins/topic.py similarity index 100% rename from csbot/plugins/topic.py rename to src/csbot/plugins/topic.py diff --git a/csbot/plugins/usertrack.py b/src/csbot/plugins/usertrack.py similarity index 100% rename from csbot/plugins/usertrack.py rename to src/csbot/plugins/usertrack.py diff --git a/csbot/plugins/webhook.py b/src/csbot/plugins/webhook.py similarity index 100% rename from csbot/plugins/webhook.py rename to src/csbot/plugins/webhook.py diff --git a/csbot/plugins/webserver.py b/src/csbot/plugins/webserver.py similarity index 100% rename from csbot/plugins/webserver.py rename to src/csbot/plugins/webserver.py diff --git a/csbot/plugins/whois.py b/src/csbot/plugins/whois.py similarity index 100% rename from csbot/plugins/whois.py rename to src/csbot/plugins/whois.py diff --git a/csbot/plugins/xkcd.py b/src/csbot/plugins/xkcd.py similarity index 82% rename from csbot/plugins/xkcd.py rename to src/csbot/plugins/xkcd.py index 909fccdd..d4b847c9 100644 --- a/csbot/plugins/xkcd.py +++ b/src/csbot/plugins/xkcd.py @@ -1,9 +1,8 @@ import html import random -import requests from ..plugin import Plugin -from ..util import simple_http_get, cap_string, is_ascii +from ..util import simple_http_get_async, cap_string, is_ascii from .linkinfo import LinkInfoResult @@ -29,7 +28,7 @@ def fix_json_unicode(data): return data -def get_info(number=None): +async def get_info(number=None): """Gets the json data for a particular comic (or the latest, if none provided). """ @@ -38,13 +37,13 @@ def get_info(number=None): else: url = "http://xkcd.com/info.0.json" - httpdata = simple_http_get(url) - if httpdata.status_code != requests.codes.ok: - return None + async with simple_http_get_async(url) as httpdata: + if httpdata.status != 200: + return None - # Only care about part of the data - httpjson = httpdata.json() - data = {key: httpjson[key] for key in ["title", "alt", "num"]} + # Only care about part of the data + httpjson = await httpdata.json() + data = {key: httpjson[key] for key in ["title", "alt", "num"]} # Unfuck up unicode strings data = fix_json_unicode(data) @@ -61,12 +60,12 @@ class xkcd(Plugin): class XKCDError(Exception): pass - def _xkcd(self, user_str): + async def _xkcd(self, user_str): """Get the url and title stuff. Returns a string of the response. """ - latest = get_info() + latest = await get_info() if not latest: raise self.XKCDError("Error getting comics") @@ -75,12 +74,12 @@ def _xkcd(self, user_str): if not user_str or user_str in {'0', 'latest', 'current', 'newest'}: requested = latest elif user_str in {'rand', 'random'}: - requested = get_info(random.randint(1, latest_num)) + requested = await get_info(random.randint(1, latest_num)) else: try: num = int(user_str) if 1 <= num <= latest_num: - requested = get_info(num) + requested = await get_info(num) else: raise self.XKCDError("Comic #{} is invalid. The latest is #{}" .format(num, latest_num)) @@ -99,14 +98,14 @@ def _xkcd(self, user_str): def linkinfo_integrate(self, linkinfo): """Handle recognised xkcd urls.""" - def page_handler(url, match): + async def page_handler(url, match): """Use the main _xkcd function, then modify the result (if success) so it looks nicer. """ # Remove leading and trailing '/' try: - response = self._xkcd(url.path.strip('/')) + response = await self._xkcd(url.path.strip('/')) return LinkInfoResult(url.geturl(), '{1} - "{2}"'.format(*response)) except self.XKCDError: return None diff --git a/csbot/plugins/youtube.py b/src/csbot/plugins/youtube.py similarity index 80% rename from csbot/plugins/youtube.py rename to src/csbot/plugins/youtube.py index 2dfad3a9..ead5d495 100644 --- a/csbot/plugins/youtube.py +++ b/src/csbot/plugins/youtube.py @@ -1,8 +1,8 @@ import datetime import urllib.parse as urlparse -import apiclient import isodate +from aiogoogle import Aiogoogle, HTTPError from ..plugin import Plugin from .linkinfo import LinkInfoResult @@ -39,11 +39,9 @@ def __init__(self, http_error): self.http_error = http_error def __str__(self): - s = '%s: %s' % (self.http_error.resp.status, self.http_error._get_reason()) - if self.http_error.resp.status == 400: - return s + ' - invalid API key?' - else: - return s + s = '%s: %s' % (self.http_error.res.status_code, + self.http_error.res.json['error']['message']) + return s class Youtube(Plugin): @@ -61,39 +59,29 @@ class Youtube(Plugin): RESPONSE = '"{title}" [{duration}] (by {uploader} at {uploaded}) | Views: {views} [{likes}]' CMD_RESPONSE = RESPONSE + ' | {link}' - #: Hook for mocking HTTP responses to Google API client - http = None - client = None - - def setup(self): - super().setup() - self.client = apiclient.discovery.build( - 'youtube', 'v3', - developerKey=self.config_get('api_key'), - http=self.http) - - def get_video_json(self, id): - response = self.client.videos()\ - .list(id=id, hl='en', part='snippet,contentDetails,statistics')\ - .execute(http=self.http) - if len(response['items']) == 0: - return None - else: - return response['items'][0] + async def get_video_json(self, id): + async with Aiogoogle(api_key=self.config_get('api_key')) as aiogoogle: + youtube_v3 = await aiogoogle.discover('youtube', 'v3') + request = youtube_v3.videos.list(id=id, hl='en', part='snippet,contentDetails,statistics') + response = await aiogoogle.as_api_key(request) + if len(response['items']) == 0: + return None + else: + return response['items'][0] - def _yt(self, url): + async def _yt(self, url): """Builds a nicely formatted version of youtube's own internal JSON""" vid_id = get_yt_id(url) if not vid_id: return None try: - json = self.get_video_json(vid_id) + json = await self.get_video_json(vid_id) if json is None: return None except (KeyError, ValueError): return None - except apiclient.errors.HttpError as e: + except HTTPError as e: # Chain our own exception that gets a more sanitised error message raise YoutubeError(e) from e @@ -155,10 +143,10 @@ def _yt(self, url): def linkinfo_integrate(self, linkinfo): """Handle recognised youtube urls.""" - def page_handler(url, match): + async def page_handler(url, match): """Handles privmsg urls.""" try: - response = self._yt(url) + response = await self._yt(url) if response: return LinkInfoResult(url.geturl(), self.RESPONSE.format(**response)) else: @@ -171,11 +159,11 @@ def page_handler(url, match): @Plugin.command('youtube') @Plugin.command('yt') - def all_hail_our_google_overlords(self, e): + async def all_hail_our_google_overlords(self, e): """I for one, welcome our Google overlords.""" try: - response = self._yt(urlparse.urlparse(e["data"])) + response = await self._yt(urlparse.urlparse(e["data"])) if not response: e.reply("Invalid video ID") else: diff --git a/csbot/plugins_broken/tell.py b/src/csbot/plugins_broken/tell.py similarity index 100% rename from csbot/plugins_broken/tell.py rename to src/csbot/plugins_broken/tell.py diff --git a/csbot/plugins_broken/users.py b/src/csbot/plugins_broken/users.py similarity index 100% rename from csbot/plugins_broken/users.py rename to src/csbot/plugins_broken/users.py diff --git a/csbot/util.py b/src/csbot/util.py similarity index 86% rename from csbot/util.py rename to src/csbot/util.py index 7022657f..3e39de11 100644 --- a/csbot/util.py +++ b/src/csbot/util.py @@ -1,8 +1,15 @@ import shlex from itertools import tee from collections import OrderedDict +import asyncio +import logging import requests +from async_generator import asynccontextmanager +import aiohttp + + +LOG = logging.getLogger(__name__) class User(object): @@ -96,6 +103,19 @@ def simple_http_get(url, stream=False): return requests.get(url, verify=False, headers=headers, stream=stream) +@asynccontextmanager +async def simple_http_get_async(url, **kwargs): + session_kwargs = { + 'headers': { + 'User-Agent': 'csbot/0.1', + }, + } + kwargs.setdefault('ssl', False) + async with aiohttp.ClientSession(**session_kwargs) as session: + async with session.get(url, **kwargs) as resp: + yield resp + + def pairwise(iterable): """Pairs elements of an iterable together, e.g. s -> (s0,s1), (s1,s2), (s2, s3), ... @@ -300,3 +320,35 @@ def __repr__(self): return '{}({})'.format(self.__class__.__name__, ', '.join('{}={!r}'.format(k, getattr(self, k)) for k in self._fields)) + + +def maybe_future(result, *, on_error=None, log=LOG, loop=None): + """Make *result* a future if possible, otherwise return None. + + If *result* is not None but also not awaitable, it is passed to *on_error* + if supplied, otherwise logged as a warning on *log*. + """ + if result is None: + return None + try: + future = asyncio.ensure_future(result, loop=loop) + except TypeError: + if on_error: + on_error(result) + else: + log.warning('maybe_future() ignoring non-awaitable result %r', result) + return None + return future + + +async def maybe_future_result(result, **kwargs): + """Get actual result from *result*. + + If *result* is awaitable, return the result of awaiting it, otherwise just + return *result*. + """ + future = maybe_future(result, **kwargs) + if future: + return await future + else: + return result diff --git a/csbot/test/__init__.py b/tests/__init__.py similarity index 100% rename from csbot/test/__init__.py rename to tests/__init__.py diff --git a/csbot/test/conftest.py b/tests/conftest.py similarity index 94% rename from csbot/test/conftest.py rename to tests/conftest.py index 7437d728..0371c8a1 100644 --- a/csbot/test/conftest.py +++ b/tests/conftest.py @@ -6,10 +6,11 @@ import pytest import aiofastforward import responses as responses_ +from aioresponses import aioresponses as aioresponses_ -from csbot import test from csbot.irc import IRCClient from csbot.core import Bot +from . import mock_open_connection @pytest.fixture @@ -54,7 +55,7 @@ async def irc_client(request, event_loop, irc_client_class, pre_irc_client, irc_ else: client = irc_client_class(loop=event_loop, **irc_client_config) # Connect fake stream reader/writer (for tests that don't need the read loop) - with test.mock_open_connection(): + with mock_open_connection(): await client.connect() # Mock all the things! @@ -105,8 +106,7 @@ def receive(self, lines): """Shortcut to push a series of lines to the client.""" if isinstance(lines, str): lines = [lines] - for l in lines: - self.client.line_received(l) + return [self.client.line_received(l) for l in lines] def assert_sent(self, lines): """Check that a list of (unicode) strings have been sent. @@ -138,7 +138,7 @@ async def run_client(event_loop, irc_client_helper): ... await irc_client_helper.receive_bytes(b":nick!user@host PRIVMSG #channel :hello\r\n") ... irc_client_helper.assert_sent('PRIVMSG #channel :what do you mean, hello?') """ - with test.mock_open_connection(): + with mock_open_connection(): # Start the client run_fut = event_loop.create_task(irc_client_helper.client.run()) await irc_client_helper.client.connected.wait() @@ -177,3 +177,9 @@ def bot(self): def responses(): with responses_.RequestsMock() as rsps: yield rsps + + +@pytest.fixture +def aioresponses(): + with aioresponses_() as m: + yield m diff --git a/csbot/test/fixtures/empty_file b/tests/fixtures/empty_file similarity index 100% rename from csbot/test/fixtures/empty_file rename to tests/fixtures/empty_file diff --git a/csbot/test/fixtures/github/github-create-20190129-215300.headers.json b/tests/fixtures/github/github-create-20190129-215300.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190129-215300.headers.json rename to tests/fixtures/github/github-create-20190129-215300.headers.json diff --git a/csbot/test/fixtures/github/github-create-20190129-215300.payload.json b/tests/fixtures/github/github-create-20190129-215300.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190129-215300.payload.json rename to tests/fixtures/github/github-create-20190129-215300.payload.json diff --git a/csbot/test/fixtures/github/github-create-20190130-101054.headers.json b/tests/fixtures/github/github-create-20190130-101054.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190130-101054.headers.json rename to tests/fixtures/github/github-create-20190130-101054.headers.json diff --git a/csbot/test/fixtures/github/github-create-20190130-101054.payload.json b/tests/fixtures/github/github-create-20190130-101054.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190130-101054.payload.json rename to tests/fixtures/github/github-create-20190130-101054.payload.json diff --git a/csbot/test/fixtures/github/github-delete-20190129-215230.headers.json b/tests/fixtures/github/github-delete-20190129-215230.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-delete-20190129-215230.headers.json rename to tests/fixtures/github/github-delete-20190129-215230.headers.json diff --git a/csbot/test/fixtures/github/github-delete-20190129-215230.payload.json b/tests/fixtures/github/github-delete-20190129-215230.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-delete-20190129-215230.payload.json rename to tests/fixtures/github/github-delete-20190129-215230.payload.json diff --git a/csbot/test/fixtures/github/github-issues-assigned-20190128-101919.headers.json b/tests/fixtures/github/github-issues-assigned-20190128-101919.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-assigned-20190128-101919.headers.json rename to tests/fixtures/github/github-issues-assigned-20190128-101919.headers.json diff --git a/csbot/test/fixtures/github/github-issues-assigned-20190128-101919.payload.json b/tests/fixtures/github/github-issues-assigned-20190128-101919.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-assigned-20190128-101919.payload.json rename to tests/fixtures/github/github-issues-assigned-20190128-101919.payload.json diff --git a/csbot/test/fixtures/github/github-issues-closed-20190128-101908.headers.json b/tests/fixtures/github/github-issues-closed-20190128-101908.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-closed-20190128-101908.headers.json rename to tests/fixtures/github/github-issues-closed-20190128-101908.headers.json diff --git a/csbot/test/fixtures/github/github-issues-closed-20190128-101908.payload.json b/tests/fixtures/github/github-issues-closed-20190128-101908.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-closed-20190128-101908.payload.json rename to tests/fixtures/github/github-issues-closed-20190128-101908.payload.json diff --git a/csbot/test/fixtures/github/github-issues-opened-20190128-101904.headers.json b/tests/fixtures/github/github-issues-opened-20190128-101904.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-opened-20190128-101904.headers.json rename to tests/fixtures/github/github-issues-opened-20190128-101904.headers.json diff --git a/csbot/test/fixtures/github/github-issues-opened-20190128-101904.payload.json b/tests/fixtures/github/github-issues-opened-20190128-101904.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-opened-20190128-101904.payload.json rename to tests/fixtures/github/github-issues-opened-20190128-101904.payload.json diff --git a/csbot/test/fixtures/github/github-issues-reopened-20190128-101912.headers.json b/tests/fixtures/github/github-issues-reopened-20190128-101912.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-reopened-20190128-101912.headers.json rename to tests/fixtures/github/github-issues-reopened-20190128-101912.headers.json diff --git a/csbot/test/fixtures/github/github-issues-reopened-20190128-101912.payload.json b/tests/fixtures/github/github-issues-reopened-20190128-101912.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-reopened-20190128-101912.payload.json rename to tests/fixtures/github/github-issues-reopened-20190128-101912.payload.json diff --git a/csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.headers.json b/tests/fixtures/github/github-issues-unassigned-20190128-101924.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.headers.json rename to tests/fixtures/github/github-issues-unassigned-20190128-101924.headers.json diff --git a/csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.payload.json b/tests/fixtures/github/github-issues-unassigned-20190128-101924.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.payload.json rename to tests/fixtures/github/github-issues-unassigned-20190128-101924.payload.json diff --git a/csbot/test/fixtures/github/github-ping-20190128-101509.headers.json b/tests/fixtures/github/github-ping-20190128-101509.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-ping-20190128-101509.headers.json rename to tests/fixtures/github/github-ping-20190128-101509.headers.json diff --git a/csbot/test/fixtures/github/github-ping-20190128-101509.payload.json b/tests/fixtures/github/github-ping-20190128-101509.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-ping-20190128-101509.payload.json rename to tests/fixtures/github/github-ping-20190128-101509.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json b/tests/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json rename to tests/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json b/tests/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json rename to tests/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.headers.json b/tests/fixtures/github/github-pull_request-closed-20190129-215221.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.headers.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215221.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.payload.json b/tests/fixtures/github/github-pull_request-closed-20190129-215221.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.payload.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215221.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.headers.json b/tests/fixtures/github/github-pull_request-closed-20190129-215329.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.headers.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215329.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.payload.json b/tests/fixtures/github/github-pull_request-closed-20190129-215329.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.payload.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215329.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.headers.json b/tests/fixtures/github/github-pull_request-opened-20190129-215304.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.headers.json rename to tests/fixtures/github/github-pull_request-opened-20190129-215304.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.payload.json b/tests/fixtures/github/github-pull_request-opened-20190129-215304.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.payload.json rename to tests/fixtures/github/github-pull_request-opened-20190129-215304.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json b/tests/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json rename to tests/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json b/tests/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json rename to tests/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json b/tests/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json rename to tests/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json b/tests/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json rename to tests/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json b/tests/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json rename to tests/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json b/tests/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json rename to tests/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json b/tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json rename to tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json b/tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json rename to tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215221.headers.json b/tests/fixtures/github/github-push-20190129-215221.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215221.headers.json rename to tests/fixtures/github/github-push-20190129-215221.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215221.payload.json b/tests/fixtures/github/github-push-20190129-215221.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215221.payload.json rename to tests/fixtures/github/github-push-20190129-215221.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215300.headers.json b/tests/fixtures/github/github-push-20190129-215300.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215300.headers.json rename to tests/fixtures/github/github-push-20190129-215300.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215300.payload.json b/tests/fixtures/github/github-push-20190129-215300.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215300.payload.json rename to tests/fixtures/github/github-push-20190129-215300.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190130-195825.headers.json b/tests/fixtures/github/github-push-20190130-195825.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190130-195825.headers.json rename to tests/fixtures/github/github-push-20190130-195825.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190130-195825.payload.json b/tests/fixtures/github/github-push-20190130-195825.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190130-195825.payload.json rename to tests/fixtures/github/github-push-20190130-195825.payload.json diff --git a/csbot/test/fixtures/github/github-release-published-20190130-101053.headers.json b/tests/fixtures/github/github-release-published-20190130-101053.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-release-published-20190130-101053.headers.json rename to tests/fixtures/github/github-release-published-20190130-101053.headers.json diff --git a/csbot/test/fixtures/github/github-release-published-20190130-101053.payload.json b/tests/fixtures/github/github-release-published-20190130-101053.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-release-published-20190130-101053.payload.json rename to tests/fixtures/github/github-release-published-20190130-101053.payload.json diff --git a/csbot/test/fixtures/google-discovery-youtube-v3.json b/tests/fixtures/google-discovery-youtube-v3.json similarity index 100% rename from csbot/test/fixtures/google-discovery-youtube-v3.json rename to tests/fixtures/google-discovery-youtube-v3.json diff --git a/csbot/test/fixtures/imgur_album_26hit.json b/tests/fixtures/imgur_album_26hit.json similarity index 100% rename from csbot/test/fixtures/imgur_album_26hit.json rename to tests/fixtures/imgur_album_26hit.json diff --git a/csbot/test/fixtures/imgur_album_myXfq.json b/tests/fixtures/imgur_album_myXfq.json similarity index 100% rename from csbot/test/fixtures/imgur_album_myXfq.json rename to tests/fixtures/imgur_album_myXfq.json diff --git a/csbot/test/fixtures/imgur_album_ysj7k.json b/tests/fixtures/imgur_album_ysj7k.json similarity index 100% rename from csbot/test/fixtures/imgur_album_ysj7k.json rename to tests/fixtures/imgur_album_ysj7k.json diff --git a/csbot/test/fixtures/imgur_credits.json b/tests/fixtures/imgur_credits.json similarity index 100% rename from csbot/test/fixtures/imgur_credits.json rename to tests/fixtures/imgur_credits.json diff --git a/csbot/test/fixtures/imgur_gallery_HNUmA0P.json b/tests/fixtures/imgur_gallery_HNUmA0P.json similarity index 100% rename from csbot/test/fixtures/imgur_gallery_HNUmA0P.json rename to tests/fixtures/imgur_gallery_HNUmA0P.json diff --git a/csbot/test/fixtures/imgur_gallery_rYRa1.json b/tests/fixtures/imgur_gallery_rYRa1.json similarity index 100% rename from csbot/test/fixtures/imgur_gallery_rYRa1.json rename to tests/fixtures/imgur_gallery_rYRa1.json diff --git a/csbot/test/fixtures/imgur_image_jSmKOXT.json b/tests/fixtures/imgur_image_jSmKOXT.json similarity index 100% rename from csbot/test/fixtures/imgur_image_jSmKOXT.json rename to tests/fixtures/imgur_image_jSmKOXT.json diff --git a/csbot/test/fixtures/imgur_image_ybgvNbM.json b/tests/fixtures/imgur_image_ybgvNbM.json similarity index 100% rename from csbot/test/fixtures/imgur_image_ybgvNbM.json rename to tests/fixtures/imgur_image_ybgvNbM.json diff --git a/csbot/test/fixtures/imgur_invalid_album_id.json b/tests/fixtures/imgur_invalid_album_id.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_album_id.json rename to tests/fixtures/imgur_invalid_album_id.json diff --git a/csbot/test/fixtures/imgur_invalid_api_key.json b/tests/fixtures/imgur_invalid_api_key.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_api_key.json rename to tests/fixtures/imgur_invalid_api_key.json diff --git a/csbot/test/fixtures/imgur_invalid_gallery.json b/tests/fixtures/imgur_invalid_gallery.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_gallery.json rename to tests/fixtures/imgur_invalid_gallery.json diff --git a/csbot/test/fixtures/imgur_invalid_image_id.json b/tests/fixtures/imgur_invalid_image_id.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_image_id.json rename to tests/fixtures/imgur_invalid_image_id.json diff --git a/csbot/test/fixtures/imgur_nsfw_album.json b/tests/fixtures/imgur_nsfw_album.json similarity index 100% rename from csbot/test/fixtures/imgur_nsfw_album.json rename to tests/fixtures/imgur_nsfw_album.json diff --git a/csbot/test/fixtures/xkcd_1.json b/tests/fixtures/xkcd_1.json similarity index 100% rename from csbot/test/fixtures/xkcd_1.json rename to tests/fixtures/xkcd_1.json diff --git a/csbot/test/fixtures/xkcd_1363.json b/tests/fixtures/xkcd_1363.json similarity index 100% rename from csbot/test/fixtures/xkcd_1363.json rename to tests/fixtures/xkcd_1363.json diff --git a/csbot/test/fixtures/xkcd_1506.json b/tests/fixtures/xkcd_1506.json similarity index 100% rename from csbot/test/fixtures/xkcd_1506.json rename to tests/fixtures/xkcd_1506.json diff --git a/csbot/test/fixtures/xkcd_2043.json b/tests/fixtures/xkcd_2043.json similarity index 100% rename from csbot/test/fixtures/xkcd_2043.json rename to tests/fixtures/xkcd_2043.json diff --git a/csbot/test/fixtures/xkcd_259.json b/tests/fixtures/xkcd_259.json similarity index 100% rename from csbot/test/fixtures/xkcd_259.json rename to tests/fixtures/xkcd_259.json diff --git a/csbot/test/fixtures/xkcd_403.json b/tests/fixtures/xkcd_403.json similarity index 100% rename from csbot/test/fixtures/xkcd_403.json rename to tests/fixtures/xkcd_403.json diff --git a/csbot/test/fixtures/xkcd_latest.json b/tests/fixtures/xkcd_latest.json similarity index 100% rename from csbot/test/fixtures/xkcd_latest.json rename to tests/fixtures/xkcd_latest.json diff --git a/csbot/test/fixtures/youtube_539OnO-YImk.json b/tests/fixtures/youtube_539OnO-YImk.json similarity index 100% rename from csbot/test/fixtures/youtube_539OnO-YImk.json rename to tests/fixtures/youtube_539OnO-YImk.json diff --git a/csbot/test/fixtures/youtube_access_not_configured.json b/tests/fixtures/youtube_access_not_configured.json similarity index 100% rename from csbot/test/fixtures/youtube_access_not_configured.json rename to tests/fixtures/youtube_access_not_configured.json diff --git a/csbot/test/fixtures/youtube_fItlK6L-khc.json b/tests/fixtures/youtube_fItlK6L-khc.json similarity index 100% rename from csbot/test/fixtures/youtube_fItlK6L-khc.json rename to tests/fixtures/youtube_fItlK6L-khc.json diff --git a/csbot/test/fixtures/youtube_flibble.json b/tests/fixtures/youtube_flibble.json similarity index 100% rename from csbot/test/fixtures/youtube_flibble.json rename to tests/fixtures/youtube_flibble.json diff --git a/csbot/test/fixtures/youtube_invalid_key.json b/tests/fixtures/youtube_invalid_key.json similarity index 100% rename from csbot/test/fixtures/youtube_invalid_key.json rename to tests/fixtures/youtube_invalid_key.json diff --git a/csbot/test/fixtures/youtube_sw4hmqVPe0E.json b/tests/fixtures/youtube_sw4hmqVPe0E.json similarity index 100% rename from csbot/test/fixtures/youtube_sw4hmqVPe0E.json rename to tests/fixtures/youtube_sw4hmqVPe0E.json diff --git a/csbot/test/fixtures/youtube_vZ_YpOvRd3o.json b/tests/fixtures/youtube_vZ_YpOvRd3o.json similarity index 100% rename from csbot/test/fixtures/youtube_vZ_YpOvRd3o.json rename to tests/fixtures/youtube_vZ_YpOvRd3o.json diff --git a/tests/test_bot.py b/tests/test_bot.py new file mode 100644 index 00000000..3f5b7c16 --- /dev/null +++ b/tests/test_bot.py @@ -0,0 +1,66 @@ +import unittest.mock as mock +import asyncio + +import pytest + +from csbot import core +from csbot.plugin import Plugin + + +class TestHookOrdering: + class Bot(core.Bot): + class MockPlugin(Plugin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handler_mock = mock.Mock(spec=callable) + + @Plugin.hook('core.message.privmsg') + async def privmsg(self, event): + await asyncio.sleep(0.5) + self.handler_mock('privmsg', event['message']) + + @Plugin.hook('core.user.quit') + def quit(self, event): + self.handler_mock('quit', event['user']) + + available_plugins = core.Bot.available_plugins.copy() + available_plugins.update( + mockplugin=MockPlugin, + ) + + CONFIG = f"""\ + [@bot] + plugins = mockplugin + """ + pytestmark = pytest.mark.bot(cls=Bot, config=CONFIG) + + @pytest.mark.asyncio + @pytest.mark.parametrize('n', list(range(1, 10))) + async def test_burst_in_order(self, bot_helper, n): + """Check that a plugin always gets messages in receive order.""" + plugin = bot_helper['mockplugin'] + users = [f':nick{i}!user{i}@host{i}' for i in range(n)] + messages = [f':{user} QUIT :*.net *.split' for user in users] + await asyncio.wait(bot_helper.receive(messages)) + assert plugin.handler_mock.mock_calls == [mock.call('quit', user) for user in users] + + @pytest.mark.asyncio + async def test_non_blocking(self, bot_helper): + plugin = bot_helper['mockplugin'] + messages = [ + ':nick0!user@host QUIT :bye', + ':nick1!user@host QUIT :bye', + ':foo!user@host PRIVMSG #channel :hello', + ':nick2!user@host QUIT :bye', + ':nick3!user@host QUIT :bye', + ':nick4!user@host QUIT :bye', + ] + await asyncio.wait(bot_helper.receive(messages)) + assert plugin.handler_mock.mock_calls == [ + mock.call('quit', 'nick0!user@host'), + mock.call('quit', 'nick1!user@host'), + mock.call('quit', 'nick2!user@host'), + mock.call('quit', 'nick3!user@host'), + mock.call('quit', 'nick4!user@host'), + mock.call('privmsg', 'hello'), + ] diff --git a/csbot/test/test_config.py b/tests/test_config.py similarity index 98% rename from csbot/test/test_config.py rename to tests/test_config.py index b7521a74..d94109cd 100644 --- a/csbot/test/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import pytest -from csbot.test import TempEnvVars +from . import TempEnvVars import csbot.plugin diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 00000000..750d8fe0 --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,765 @@ +import unittest +from unittest import mock +import datetime +import collections.abc +from collections import defaultdict +from functools import partial +import asyncio + +import pytest + +import csbot.events + + +class TestImmediateEventRunner(unittest.TestCase): + def setUp(self): + self.runner = csbot.events.ImmediateEventRunner(self.handle_event) + self.handled_events = [] + + def tearDown(self): + self.runner = None + self.handled_events = None + + def handle_event(self, event): + """Record objects passed through the event handler in order. If they + are callable, call them.""" + self.handled_events.append(event) + if isinstance(event, collections.abc.Callable): + event() + + def test_values(self): + """Check that basic values are passed through the event queue + unmolested.""" + # Test that things actually get through + self.runner.post_event('foo') + self.assertEqual(self.handled_events, ['foo']) + # The event runner doesn't care what it's passing through + for x in ['bar', 1.3, None, object]: + self.runner.post_event(x) + self.assertIs(self.handled_events[-1], x) + + def test_event_chain(self): + """Check that chains of events get handled.""" + def f1(): + self.runner.post_event(f2) + + def f2(): + self.runner.post_event(f3) + + def f3(): + pass + + self.runner.post_event(f1) + self.assertEqual(self.handled_events, [f1, f2, f3]) + + def test_event_tree(self): + """Check that trees of events are handled breadth-first.""" + def f1(): + self.runner.post_event(f2) + self.runner.post_event(f3) + + def f2(): + self.runner.post_event(f4) + + def f3(): + self.runner.post_event(f5) + self.runner.post_event(f6) + + def f4(): + self.runner.post_event(f3) + + def f5(): + pass + + def f6(): + pass + + self.runner.post_event(f1) + self.assertEqual(self.handled_events, + [f1, f2, f3, f4, f5, f6, f3, f5, f6]) + + def test_exception_recovery(self): + """Check that exceptions propagate out of the event runner but don't + leave it broken. + + (In an early version of ImmediateEventRunner, an exception would leave + the runner's queue non-empty and new root events would accumulate + instead of being processed.) + """ + def f1(): + self.runner.post_event(f2) + raise Exception() + + def f2(): + pass + + def f3(): + self.runner.post_event(f4) + + def f4(): + pass + + self.assertRaises(Exception, self.runner.post_event, f1) + self.assertEqual(self.handled_events, [f1]) + self.runner.post_event(f3) + self.assertEqual(self.handled_events, [f1, f3, f4]) + + +@pytest.fixture +def async_runner(event_loop): + def handle_event(event): + if asyncio.iscoroutinefunction(event): + return [event()] + else: + return [] + + obj = mock.Mock() + obj.handle_event = mock.Mock(wraps=handle_event) + obj.runner = csbot.events.AsyncEventRunner(obj.handle_event, event_loop) + obj.exception_handler = mock.Mock(wraps=event_loop.get_exception_handler()) + event_loop.set_exception_handler(obj.exception_handler) + + return obj + + +class TestAsyncEventRunner: + @pytest.mark.asyncio + def test_values(self, async_runner): + """Check that basic values are passed through the event queue + unmolested.""" + # Test that things actually get through + yield from async_runner.runner.post_event('foo') + assert async_runner.handle_event.call_args_list == [mock.call('foo')] + # The event runner doesn't care what it's passing through + for x in ['bar', 1.3, None, object]: + yield from async_runner.runner.post_event(x) + assert async_runner.handle_event.call_args[0][0] is x + + @pytest.mark.asyncio + def test_event_chain(self, async_runner): + """Check that chains of events get handled.""" + @asyncio.coroutine + def f1(): + async_runner.runner.post_event(f2) + + @asyncio.coroutine + def f2(): + async_runner.runner.post_event(f3) + + @asyncio.coroutine + def f3(): + pass + + yield from async_runner.runner.post_event(f1) + assert async_runner.handle_event.call_count == 3 + async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2), mock.call(f3)]) + + @pytest.mark.asyncio(allow_unhandled_exception=True) + def test_exception_recovery(self, async_runner): + """Check that exceptions are handled but don't block other tasks or + leave the runner in a broken state. + """ + @asyncio.coroutine + def f1(): + async_runner.runner.post_event(f2) + raise Exception() + + @asyncio.coroutine + def f2(): + pass + + @asyncio.coroutine + def f3(): + async_runner.runner.post_event(f4) + + @asyncio.coroutine + def f4(): + pass + + assert async_runner.exception_handler.call_count == 0 + yield from async_runner.runner.post_event(f1) + assert async_runner.exception_handler.call_count == 1 + async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2)]) + #self.assertEqual(set(self.handled_events), {f1, f2}) + yield from async_runner.runner.post_event(f3) + assert async_runner.exception_handler.call_count == 1 + async_runner.handle_event.assert_has_calls([mock.call(f1), mock.call(f2), mock.call(f3), mock.call(f4)]) + #self.assertEqual(set(self.handled_events), {f1, f2, f3, f4}) + + +@pytest.mark.asyncio +class TestHybridEventRunner: + class EventHandler: + def __init__(self): + self.handlers = defaultdict(list) + + def add(self, e, f=None): + if f is None: + return partial(self.add, e) + else: + self.handlers[e].append(f) + + def __call__(self, e): + return self.handlers[e] + + @pytest.fixture + def event_runner(self, event_loop): + handler = self.EventHandler() + obj = mock.Mock() + obj.add_handler = handler.add + obj.get_handlers = mock.Mock(wraps=handler) + obj.runner = csbot.events.HybridEventRunner(obj.get_handlers, event_loop) + obj.exception_handler = mock.Mock(wraps=event_loop.get_exception_handler()) + event_loop.set_exception_handler(obj.exception_handler) + return obj + + async def test_values(self, event_runner): + """Check that basic values are passed through the event queue unmolested.""" + # Test that things actually get through + await event_runner.runner.post_event('foo') + assert event_runner.get_handlers.call_args_list == [mock.call('foo')] + # The event runner doesn't care what it's passing through + for x in ['bar', 1.3, None, object]: + await event_runner.runner.post_event(x) + print(event_runner.get_handlers.call_args) + assert event_runner.get_handlers.call_args == mock.call(x) + + async def test_event_chain_synchronous(self, event_runner): + """Check that an entire event chain runs (synchronously). + + All handlers for an event should be run before the next event, and any events that occur + during an event handler should also be processed before the initial `post_event()` future + has a result. + """ + complete = [] + + @event_runner.add_handler('a') + def a(_): + event_runner.runner.post_event('b') + complete.append('a') + + @event_runner.add_handler('b') + def b1(_): + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('b') + def b3(_): + event_runner.runner.post_event('e') + complete.append('b3') + + @event_runner.add_handler('c') + def c(_): + event_runner.runner.post_event('f') + complete.append('c') + + @event_runner.add_handler('d') + def d(_): + complete.append('d') + + @event_runner.add_handler('e') + def e(_): + complete.append('e') + + await event_runner.runner.post_event('a') + assert event_runner.get_handlers.mock_calls == [ + # Initial event + mock.call('a'), + # Event resulting from handler for 'a' + mock.call('b'), + # Ensure all handlers for 'b' finished ... + mock.call('c'), + mock.call('d'), + mock.call('e'), + # ... before first handler for 'c' + mock.call('f'), + ] + assert complete == ['a', 'b1', 'b2', 'b3', 'c', 'd', 'e'] + + async def test_event_chain_asynchronous(self, event_loop, event_runner): + """Check that an entire event chain runs (asynchronously). + + Any events that occur during an event handler should be processed before the initial + `post_event()` future has a result. + """ + events = [asyncio.Event(loop=event_loop) for _ in range(2)] + complete = [] + + @event_runner.add_handler('a') + async def a1(_): + complete.append('a1') + + @event_runner.add_handler('a') + async def a2(_): + await events[0].wait() + event_runner.runner.post_event('b') + complete.append('a2') + + @event_runner.add_handler('b') + async def b1(_): + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + async def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('b') + async def b3(_): + await events[1].wait() + event_runner.runner.post_event('e') + complete.append('b3') + + @event_runner.add_handler('c') + async def c(_): + event_runner.runner.post_event('f') + complete.append('c') + + @event_runner.add_handler('d') + async def d(_): + complete.append('d') + + @event_runner.add_handler('e') + async def e(_): + complete.append('e') + + # Post the first event and allow some tasks to run: + # - should have a post_event('a') call + # - a1 should complete, a2 is blocked on events[0] + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + ] + assert complete == ['a1'] + + # Unblock a2 and allow some tasks to run: + # - a2 should complete + # - post_event('b') should be called (by a2) + # - b1 and b2 should complete, b3 is blocked on events[1] + # - post_event('c') and post_event('d') should be called (by b1 and b2) + # - c should complete + # - post_event('f') should be called (by c) + # - d should complete + events[0].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('c'), + mock.call('d'), + mock.call('f'), + ] + assert complete == ['a1', 'a2', 'b1', 'b2', 'c', 'd'] + + # Unblock b3 and allow some tasks to run: + # - b3 should complete + # - post_event('e') should be called (by b3) + # - e should complete + # - future should complete, because no events or tasks remain pending + events[1].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('c'), + mock.call('d'), + mock.call('f'), + mock.call('e'), + ] + assert complete == ['a1', 'a2', 'b1', 'b2', 'c', 'd', 'b3', 'e'] + + async def test_event_chain_hybrid(self, event_loop, event_runner): + """Check that an entire event chain runs (mix of sync and async handlers). + + Synchronous handlers complete before asynchronous handlers. Synchronous handlers for an + event all run before synchronous handlers for the next event, but asynchronous handers can + run out-of-order. + """ + events = [asyncio.Event(loop=event_loop) for _ in range(2)] + complete = [] + + @event_runner.add_handler('a') + def a1(_): + complete.append('a1') + + @event_runner.add_handler('a') + async def a2(_): + await events[0].wait() + event_runner.runner.post_event('b') + complete.append('a2') + + @event_runner.add_handler('b') + async def b1(_): + await events[1].wait() + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('c') + def c1(_): + complete.append('c1') + + @event_runner.add_handler('c') + async def c2(_): + complete.append('c2') + + @event_runner.add_handler('d') + async def d1(_): + complete.append('d1') + + @event_runner.add_handler('d') + def d2(_): + complete.append('d2') + + # Post the first event and allow some tasks to run: + # - post_event('a') should be called (initial) + # - a1 should complete, a2 is blocked on events[0] + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + ] + assert complete == ['a1'] + + # Unblock a2 and allow some tasks to run: + # - a2 should complete + # - post_event('b') should be called (by a2) + # - b2 should complete, b1 is blocked on events[1] + # - post_event('d') should be called + # - d2 should complete (synchronous phase) + # - d1 should complete (asynchronous phase) + events[0].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('d'), + ] + assert complete == ['a1', 'a2', 'b2', 'd2', 'd1'] + + # Unblock b1 and allow some tasks to run: + # - b1 should complete + # - post_event('c') should be called (by b1) + # - c1 should complete (synchronous phase) + # - c2 should complete (asynchronous phase) + # - future should complete, because no events or tasks remain pending + events[1].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert event_runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('d'), + mock.call('c'), + ] + assert complete == ['a1', 'a2', 'b2', 'd2', 'd1', 'b1', 'c1', 'c2'] + + async def test_overlapping_root_events(self, event_loop, event_runner): + """Check that overlapping events get the same future.""" + events = [asyncio.Event(loop=event_loop) for _ in range(1)] + complete = [] + + @event_runner.add_handler('a') + async def a(_): + await events[0].wait() + complete.append('a') + + @event_runner.add_handler('b') + async def b(_): + complete.append('b') + + # Post the first event and allow tasks to run: + # - a is blocked on events[0] + f1 = event_runner.runner.post_event('a') + await asyncio.wait({f1}, loop=event_loop, timeout=0.1) + assert not f1.done() + assert complete == [] + + # Post the second event and allow tasks to run: + # - b completes + # - a is still blocked on events[0] + # - f1 and f2 are not done, because they're for the same run loop, and a is still blocked + f2 = event_runner.runner.post_event('b') + await asyncio.wait({f2}, loop=event_loop, timeout=0.1) + assert not f2.done() + assert not f1.done() + assert complete == ['b'] + + # Unblock a and allow tasks to run: + # - a completes + # - f1 and f2 are both done, because the run loop has finished + events[0].set() + await asyncio.wait([f1, f2], loop=event_loop, timeout=0.1) + assert f1.done() + assert f2.done() + assert complete == ['b', 'a'] + + # (Maybe remove this - not essential that they're the same future, only that they complete together) + assert f2 is f1 + + async def test_non_overlapping_root_events(self, event_loop, event_runner): + """Check that non-overlapping events get new futures.""" + complete = [] + + @event_runner.add_handler('a') + async def a(_): + complete.append('a') + + @event_runner.add_handler('b') + async def b(_): + complete.append('b') + + f1 = event_runner.runner.post_event('a') + await asyncio.wait({f1}, loop=event_loop, timeout=0.1) + assert f1.done() + assert complete == ['a'] + + f2 = event_runner.runner.post_event('b') + assert not f2.done() + assert f2 is not f1 + await asyncio.wait({f2}, loop=event_loop, timeout=0.1) + assert f2.done() + assert complete == ['a', 'b'] + + @pytest.mark.asyncio(allow_unhandled_exception=True) + async def test_exception_recovery(self, event_loop, event_runner): + complete = [] + + @event_runner.add_handler('a') + def a1(_): + raise Exception('a1') + complete.append('a1') + + @event_runner.add_handler('a') + def a2(_): + complete.append('a2') + + @event_runner.add_handler('a') + async def a3(_): + raise Exception('a3') + complete.append('a3') + + @event_runner.add_handler('a') + async def a4(_): + event_runner.runner.post_event('b') + complete.append('a4') + + @event_runner.add_handler('b') + def b1(_): + raise Exception('b1') + complete.append('b1') + + @event_runner.add_handler('b') + async def b2(_): + complete.append('b2') + + assert event_runner.exception_handler.call_count == 0 + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert future.exception() is None + assert event_runner.runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + ] + assert complete == ['a2', 'a4', 'b2'] + assert event_runner.exception_handler.call_count == 3 + + # Check that exception handler calls have the correct event context + assert event_runner.exception_handler.mock_calls[0][1][1]['csbot_event'] == 'a' + assert event_runner.exception_handler.mock_calls[1][1][1]['csbot_event'] == 'a' + assert event_runner.exception_handler.mock_calls[2][1][1]['csbot_event'] == 'b' + + +class TestEvent(unittest.TestCase): + class DummyBot(object): + pass + + def _assert_events_equal(self, e1, e2, bot=True, + event_type=True, datetime=True, data=True): + """Test helper for comparing two events. ``<property>=False`` disables + checking that property of the events.""" + if bot: + self.assertIs(e1.bot, e2.bot) + if event_type: + self.assertEqual(e1.event_type, e2.event_type) + if datetime: + self.assertEqual(e1.datetime, e2.datetime) + if data: + for k in list(e1.keys()) + list(e2.keys()): + self.assertEqual(e1[k], e2[k]) + + def test_create(self): + # Test data + data = {'a': 1, 'b': 2, 'c': None} + dt = datetime.datetime.now() + bot = self.DummyBot() + + # Create the event + e = csbot.events.Event(bot, 'event.type', data) + # Check that the event's datetime can be reasonably considered "now" + self.assertTrue(dt <= e.datetime) + self.assertTrue(abs(e.datetime - dt) < datetime.timedelta(seconds=1)) + # Check that the bot, event type and data made it through + self.assertIs(e.bot, bot) + self.assertEqual(e.event_type, 'event.type') + for k, v in data.items(): + self.assertEqual(e[k], v) + + def test_extend(self): + # Test data + data1 = {'a': 1, 'b': 2, 'c': None} + data2 = {'c': 'foo', 'd': 'bar'} + et1 = 'event.type' + et2 = 'other.event' + bot = self.DummyBot() + + # Create an event + e1 = csbot.events.Event(bot, et1, data1) + + # Unchanged event + e2 = csbot.events.Event.extend(e1) + self._assert_events_equal(e1, e2) + + # Change event type only + e3 = csbot.events.Event.extend(e2, et2) + # Check the event type was changed + self.assertEqual(e3.event_type, et2) + # Check that everything else stayed the same + self._assert_events_equal(e1, e3, event_type=False) + + # Change the event type and data + e4 = csbot.events.Event.extend(e1, et2, data2) + # Check the event type was changed + self.assertEqual(e4.event_type, et2) + # Check the data was updated + for k in data1: + if k not in data2: + self.assertEqual(e4[k], data1[k]) + for k in data2: + self.assertEqual(e4[k], data2[k]) + # Check that everything else stayed the same + self._assert_events_equal(e1, e4, event_type=False, data=False) + + +class TestCommandEvent(unittest.TestCase): + def setUp(self): + self.nick = 'csbot' + + def _check_valid_command(self, message, prefix, command, data): + """Test helper for checking the result of parsing a command from a + message.""" + e = csbot.events.Event(None, 'test.event', {'message': message}) + c = csbot.events.CommandEvent.parse_command(e, prefix, self.nick) + self.assertEqual(c['command'], command) + self.assertEqual(c['data'], data) + return c + + def _check_invalid_command(self, message, prefix): + """Test helper for verifying that an invalid command is not + interpreted as a valid command.""" + e = csbot.events.Event(None, 'test.event', {'message': message}) + c = csbot.events.CommandEvent.parse_command(e, prefix, self.nick) + self.assertIs(c, None) + return c + + def test_parse_command(self): + # --> Test variations on command and data text with no prefix involvement + # Just a command + self._check_valid_command('testcommand', '', + 'testcommand', '') + # Command and data + self._check_valid_command('test command data', '', + 'test', 'command data') + # Leading/trailing spaces are ignored + self._check_valid_command(' test command', '', 'test', 'command') + self._check_valid_command('test command ', '', 'test', 'command') + self._check_valid_command(' test command ', '', 'test', 'command') + # Non-alphanumeric commands + self._check_valid_command('!#?$ you !', '', '!#?$', 'you !') + + # --> Test what happens with a command prefix + # Not a command + self._check_invalid_command('just somebody talking', '!') + # A simple command + self._check_valid_command('!hello', '!', 'hello', '') + # ... with data + self._check_valid_command('!hello there', '!', 'hello', 'there') + # ... and repeated prefix + self._check_valid_command('!hello !there everybody', '!', + 'hello', '!there everybody') + # Leading spaces + self._check_valid_command(' !hello', '!', 'hello', '') + # Spaces separating the prefix from the command shouldn't trigger it + self._check_invalid_command('! hello', '!') + # The prefix can be part of the command if repeated + self._check_valid_command('!!hello', '!', '!hello', '') + self._check_valid_command('!!', '!', '!', '') + + # --> Test a longer prefix + # As long as it is a prefix of the first "part", should be fine + self._check_valid_command('dosomething now', 'do', 'something', 'now') + # ... but if there's a space in between it's not a command any more + self._check_invalid_command('do something now', 'do') + + # --> Test unicode + # Unicode prefix + self._check_valid_command('\u0CA0test', '\u0CA0', 'test', '') + # Shouldn't match part of a UTF8 multibyte sequence: \u0CA0 = \xC2\xA3 + self._check_invalid_command('\u0CA0test', '\xC2') + # Unicode command + self._check_valid_command('!\u0CA0_\u0CA0', '!', '\u0CA0_\u0CA0', '') + + # Test "conversational", i.e. mentioned by nick + self._check_valid_command('csbot: do something', '!', 'do', 'something') + self._check_valid_command(' csbot, do something ', '!', 'do', 'something') + self._check_valid_command('csbot:do something', '!', 'do', 'something') + self._check_invalid_command('csbot do something', '!') + + def test_arguments(self): + """Test argument grouping/parsing. These tests are pretty much just + testing :func:`csbot.util.parse_arguments`, which should have its own + tests.""" + # No arguments + c = self._check_valid_command('!foo', '!', 'foo', '') + self.assertEqual(c.arguments(), []) + # Some simple arguments + c = self._check_valid_command('!foo bar baz', '!', 'foo', 'bar baz') + self.assertEqual(c.arguments(), ['bar', 'baz']) + # ... with extra spaces + c = self._check_valid_command('!foo bar baz ', '!', + 'foo', 'bar baz') + self.assertEqual(c.arguments(), ['bar', 'baz']) + # Forced argument grouping with quotes + c = self._check_valid_command('!foo "bar baz"', '!', + 'foo', '"bar baz"') + self.assertEqual(c.arguments(), ['bar baz']) + # ... with extra spaces + c = self._check_valid_command('!foo "bar baz " ', '!', + 'foo', '"bar baz "') + self.assertEqual(c.arguments(), ['bar baz ']) + # Escaped quote preserved + c = self._check_valid_command(r'!foo ba\"r', '!', 'foo', r'ba\"r') + self.assertEqual(c.arguments(), ['ba"r']) + # Unmatched quotes break + c = self._check_valid_command('!foo ba"r', '!', 'foo', 'ba"r') + self.assertRaises(ValueError, c.arguments) + # No mangling in the command part + c = self._check_valid_command('!"foo bar', '!', '"foo', 'bar') + c = self._check_valid_command('"foo bar', '"', 'foo', 'bar') diff --git a/csbot/test/test_irc.py b/tests/test_irc.py similarity index 79% rename from csbot/test/test_irc.py rename to tests/test_irc.py index f77ac04a..0990b94e 100644 --- a/csbot/test/test_irc.py +++ b/tests/test_irc.py @@ -3,7 +3,7 @@ import pytest -from csbot.test import mock_open_connection, mock_open_connection_paused +from . import mock_open_connection, mock_open_connection_paused from csbot.irc import IRCMessage, IRCParseError, IRCUser @@ -132,8 +132,9 @@ def irc_client_config(self): 'client_ping_interval': 3, } - @pytest.mark.asyncio(foo='bar') + @pytest.mark.asyncio async def test_client_PING(self, fast_forward, run_client): + """Check that client PING commands are sent at the expected interval.""" run_client.reset_mock() run_client.client.send_line.assert_not_called() # Advance time, test that a ping was sent @@ -161,6 +162,39 @@ async def test_client_PING(self, fast_forward, run_client): mock.call('PING 5'), ] + @pytest.mark.asyncio + async def test_client_PING_only_when_needed(self, fast_forward, run_client): + """Check that client PING commands are sent relative to the last received message.""" + run_client.reset_mock() + run_client.client.send_line.assert_not_called() + # Advance time to just before the second PING, check that the first PING was sent + await fast_forward(5) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + ] + # Receive a message, this should reset the PING timer + run_client.receive(':nick!user@host PRIVMSG #channel :foo') + # Advance time to just after when the second PING would happen without any messages + # received, check that still only one PING was sent + await fast_forward(2) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + ] + # Advance time to 4 seconds after the last message was received, and check that another + # PING has now been sent + await fast_forward(2) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + mock.call('PING 2'), + ] + # Disconnect, advance time, test that no more pings were sent + run_client.client.disconnect() + await fast_forward(12) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + mock.call('PING 2'), + ] + def test_PING_PONG(irc_client_helper): irc_client_helper.receive('PING :i.am.a.server') @@ -307,6 +341,92 @@ def test_parse_failure(irc_client_helper): irc_client_helper.receive('') +@pytest.mark.asyncio +async def test_wait_for_success(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + IRCMessage(None, 'PING', ['2'], 'PING', 'PING :2'), + ] + + mock_predicate = mock.Mock(return_value=(False, None)) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) + + # Predicate is called, but future is not resolved + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert not fut_mock.done() + + # Predicate is called, and future is resolved with matching message + mock_predicate.return_value = (True, 'foo') + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + mock.call(messages[1]), + ] + assert fut_mock.done() + assert fut_mock.result() == 'foo' + + # Predicate is not called, because once resolved it was removed + irc_client_helper.receive(messages[2].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + mock.call(messages[1]), + ] + + +@pytest.mark.asyncio +async def test_wait_for_cancelled(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + ] + + mock_predicate = mock.Mock(return_value=(False, None)) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) + + # Predicate is called, but future is not resolved + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert not fut_mock.done() + + # Predicate is not called, because future was cancelled + fut_mock.cancel() + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + + +@pytest.mark.asyncio +async def test_wait_for_exception(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + ] + + mock_predicate = mock.Mock(side_effect=Exception()) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) + + # Predicate is called, but future has exception + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert fut_mock.done() + assert fut_mock.exception() is not None + + # Predicate is not called, because future is already done + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + + # Test that calling various commands causes the appropriate messages to be sent to the server def test_set_nick(irc_client_helper): diff --git a/csbot/test/test_plugin_auth.py b/tests/test_plugin_auth.py similarity index 100% rename from csbot/test/test_plugin_auth.py rename to tests/test_plugin_auth.py diff --git a/csbot/test/test_plugin_calc.py b/tests/test_plugin_calc.py similarity index 100% rename from csbot/test/test_plugin_calc.py rename to tests/test_plugin_calc.py diff --git a/csbot/test/test_plugin_github.py b/tests/test_plugin_github.py similarity index 99% rename from csbot/test/test_plugin_github.py rename to tests/test_plugin_github.py index 8efaec69..97e98d84 100644 --- a/csbot/test/test_plugin_github.py +++ b/tests/test_plugin_github.py @@ -4,8 +4,8 @@ import asynctest from csbot import core -from csbot.test import read_fixture_file -from csbot.test.test_plugin_webserver import WebServer +from . import read_fixture_file +from .test_plugin_webserver import WebServer class Bot(core.Bot): @@ -54,7 +54,7 @@ class TestGitHubPlugin: fmt.issue_text = {issue[title]} ({issue[html_url]}) fmt.pr_num = PR #{pull_request[number]} fmt.pr_text = {pull_request[title]} ({pull_request[html_url]}) - + # Format strings for specific events fmt/create = {fmt.source} created {ref_type} {ref} ({repository[html_url]}/tree/{ref}) fmt/delete = {fmt.source} deleted {ref_type} {ref} @@ -67,7 +67,7 @@ class TestGitHubPlugin: fmt/push/pushed = {fmt.source} pushed {count} new commit(s) to {short_ref}: {compare} fmt/push/forced = {fmt.source} updated {short_ref}: {compare} fmt/release/* = {fmt.source} {event_subtype} release {release[name]}: {release[html_url]} - + [github/alanbriolat/csbot-webhook-test] notify = #mychannel """ @@ -217,7 +217,7 @@ async def test_behaviour(self, bot_helper, client, fixture_file, expected): [webhook] url_secret = test_url [github] -secret = +secret = """) async def test_signature_ignored(bot_helper, client): """X-Hub-Signature invalid, but secret is blank, so not verified and handler called""" diff --git a/csbot/test/test_plugin_helix.py b/tests/test_plugin_helix.py similarity index 100% rename from csbot/test/test_plugin_helix.py rename to tests/test_plugin_helix.py diff --git a/csbot/test/test_plugin_imgur.py b/tests/test_plugin_imgur.py similarity index 79% rename from csbot/test/test_plugin_imgur.py rename to tests/test_plugin_imgur.py index dfe379fb..bc26f4fc 100644 --- a/csbot/test/test_plugin_imgur.py +++ b/tests/test_plugin_imgur.py @@ -1,6 +1,6 @@ import pytest -from csbot.test import read_fixture_file +from . import read_fixture_file test_cases = [ @@ -164,20 +164,13 @@ """) -@pytest.fixture -def pre_irc_client(responses): - # imgurpython calls this on init to get initial rate limits - responses.add(responses.GET, 'https://api.imgur.com/3/credits', - status=200, body=read_fixture_file('imgur_credits.json'), - content_type='application/json') - - +@pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", test_cases) -def test_integration(bot_helper, responses, url, api_url, status, content_type, fixture, title): - responses.add(responses.GET, api_url, status=status, - body=read_fixture_file(fixture), - content_type=content_type) - result = bot_helper['linkinfo'].get_link_info(url) +async def test_integration(bot_helper, aioresponses, url, api_url, status, content_type, fixture, title): + aioresponses.get(api_url, status=status, + body=read_fixture_file(fixture), + content_type=content_type) + result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error else: @@ -185,12 +178,13 @@ def test_integration(bot_helper, responses, url, api_url, status, content_type, assert title == result.text +@pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", nsfw_test_cases) -def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_type, fixture, title): - responses.add(responses.GET, api_url, status=status, - body=read_fixture_file(fixture), - content_type=content_type) - result = bot_helper['linkinfo'].get_link_info(url) +async def test_integration_nsfw(bot_helper, aioresponses, url, api_url, status, content_type, fixture, title): + aioresponses.get(api_url, status=status, + body=read_fixture_file(fixture), + content_type=content_type) + result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error else: @@ -198,9 +192,8 @@ def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_t assert title == result.text -def test_invalid_URL(bot_helper, responses): +@pytest.mark.asyncio +async def test_invalid_URL(bot_helper, aioresponses): """Test that an unrecognised URL never even results in a request.""" - responses.reset() # Drop requests used/made during plugin setup - result = bot_helper['linkinfo'].get_link_info('http://imgur.com/invalid/url') + result = await bot_helper['linkinfo'].get_link_info('http://imgur.com/invalid/url') assert result.is_error - assert len(responses.calls) == 0 diff --git a/tests/test_plugin_linkinfo.py b/tests/test_plugin_linkinfo.py new file mode 100644 index 00000000..38560568 --- /dev/null +++ b/tests/test_plugin_linkinfo.py @@ -0,0 +1,305 @@ +# coding=utf-8 +from lxml.etree import LIBXML_VERSION +import unittest.mock as mock +import asyncio + +import pytest +import asynctest.mock +import aiohttp +from aioresponses import CallbackResult + +from csbot.plugin import Plugin +import csbot.core + + +#: Test encoding handling; tests are (url, content-type, body, expected_title) +encoding_test_cases = [ + # (These test case are synthetic, to test various encoding scenarios) + + # UTF-8 with Content-Type header encoding only + ( + "http://example.com/utf8-content-type-only", + "text/html; charset=utf-8", + b"<html><head><title>EM DASH \xe2\x80\x94 —", + 'EM DASH \u2014 \u2014' + ), + # UTF-8 with meta http-equiv encoding only + ( + "http://example.com/utf8-meta-http-equiv-only", + "text/html", + (b'' + b'EM DASH \xe2\x80\x94 —'), + 'EM DASH \u2014 \u2014' + ), + # UTF-8 with XML encoding declaration only + ( + "http://example.com/utf8-xml-encoding-only", + "text/html", + (b'' + b'EM DASH \xe2\x80\x94 —'), + 'EM DASH \u2014 \u2014' + ), + + # (The following are real test cases the bot has barfed on in the past) + + # Content-Type encoding, XML encoding declaration *and* http-equiv are all + # present (but no UTF-8 in title). If we give lxml a decoded string with + # the XML encoding declaration it complains. + ( + "http://www.w3.org/TR/REC-xml/", + "text/html; charset=utf-8", + b""" + + + + + + Extensible Markup Language (XML) 1.0 (Fifth Edition) + + + + + + + """, + 'Extensible Markup Language (XML) 1.0 (Fifth Edition)' + ), + # No Content-Type encoding, but has http-equiv encoding. Has a mix of + # UTF-8 literal em-dash and HTML entity em-dash - both should be output as + # unicode em-dash. + ( + "http://docs.python.org/2/library/logging.html", + "text/html", + b""" + + + + + 15.7. logging \xe2\x80\x94 Logging facility for Python — Python v2.7.3 documentation + + + + + + + """, + '15.7. logging \u2014 Logging facility for Python \u2014 Python v2.7.3 documentation' + ), + ( + "http://example.com/invalid-charset", + "text/html; charset=utf-flibble", + b'Flibble', + 'Flibble' + ), +] + +# Add HTML5 test-cases if libxml2 is new enough ( encoding +# detection was added in 2.8.0) +if LIBXML_VERSION >= (2, 8, 0): + encoding_test_cases += [ + # UTF-8 with meta charset encoding only + ( + "http://example.com/utf8-meta-charset-only", + "text/html", + (b'' + b'EM DASH \xe2\x80\x94 —'), + 'EM DASH \u2014 \u2014' + ), + ] + + +error_test_cases = [ + ( + "http://example.com/empty-title-tag", + "text/html", + b'', + ), + ( + "http://example.com/whitespace-title-tag", + "text/html", + b' ', + ), + ( + "http://example.com/no-root-element", + "text/html", + b'foo') + aioresponses.get('http://example.com/', callback=handler) + + futures = bot_helper.receive([ + ':nick!user@host PRIVMSG #channel :a', + ':nick!user@host PRIVMSG #channel :http://example.com/', + ':nick!user@host PRIVMSG #channel :b', + ]) + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert bot_helper['mockplugin'].handler_mock.mock_calls == [ + mock.call('a'), + mock.call('http://example.com/'), + mock.call('b'), + ] + bot_helper.client.send_line.assert_not_called() + + event.set() + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert all(f.done() for f in futures) + bot_helper.client.send_line.assert_has_calls([ + mock.call('NOTICE #channel :foo'), + ]) + + @pytest.mark.asyncio + async def test_non_blocking_command(self, event_loop, bot_helper, aioresponses): + bot_helper.reset_mock() + + event = asyncio.Event(loop=event_loop) + + async def handler(url, **kwargs): + await event.wait() + return CallbackResult(status=200, content_type='application/octet-stream', + body=b'foo') + + aioresponses.get('http://example.com/', callback=handler) + + futures = bot_helper.receive([ + ':nick!user@host PRIVMSG #channel :a', + ':nick!user@host PRIVMSG #channel :!link http://example.com/', + ':nick!user@host PRIVMSG #channel :b', + ]) + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert bot_helper['mockplugin'].handler_mock.mock_calls == [ + mock.call('a'), + mock.call('!link http://example.com/'), + mock.call('b'), + ] + bot_helper.client.send_line.assert_not_called() + + event.set() + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert all(f.done() for f in futures) + bot_helper.client.send_line.assert_has_calls([ + mock.call('NOTICE #channel :Error: Content-Type not HTML-ish: ' + 'application/octet-stream (http://example.com/)'), + ]) diff --git a/csbot/test/test_plugin_usertrack.py b/tests/test_plugin_usertrack.py similarity index 100% rename from csbot/test/test_plugin_usertrack.py rename to tests/test_plugin_usertrack.py diff --git a/csbot/test/test_plugin_webhook.py b/tests/test_plugin_webhook.py similarity index 97% rename from csbot/test/test_plugin_webhook.py rename to tests/test_plugin_webhook.py index 68a4c31b..405ece45 100644 --- a/csbot/test/test_plugin_webhook.py +++ b/tests/test_plugin_webhook.py @@ -4,7 +4,7 @@ from csbot import core from csbot.plugin import Plugin -from csbot.test.test_plugin_webserver import WebServer +from .test_plugin_webserver import WebServer class WebhookTest(Plugin): diff --git a/csbot/test/test_plugin_webserver.py b/tests/test_plugin_webserver.py similarity index 100% rename from csbot/test/test_plugin_webserver.py rename to tests/test_plugin_webserver.py diff --git a/csbot/test/test_plugin_whois.py b/tests/test_plugin_whois.py similarity index 100% rename from csbot/test/test_plugin_whois.py rename to tests/test_plugin_whois.py diff --git a/csbot/test/test_plugin_xkcd.py b/tests/test_plugin_xkcd.py similarity index 67% rename from csbot/test/test_plugin_xkcd.py rename to tests/test_plugin_xkcd.py index 844c647a..1b52457b 100644 --- a/csbot/test/test_plugin_xkcd.py +++ b/tests/test_plugin_xkcd.py @@ -2,7 +2,7 @@ import pytest -from csbot.test import read_fixture_file +from . import read_fixture_file #: Tests are (number, url, content-type, fixture, expected) @@ -87,48 +87,54 @@ """) class TestXKCDPlugin: @pytest.fixture - def populate_responses(self, bot_helper, responses): - """Populate all data into responses, don't assert that every request is fired.""" - responses.assert_all_requests_are_fired = False + def populate_responses(self, bot_helper, aioresponses): + """Populate all data into responses.""" for num, url, content_type, fixture, expected in json_test_cases: - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) + aioresponses.add(url, body=read_fixture_file(fixture), + content_type=content_type) + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") @pytest.mark.parametrize("num, url, content_type, fixture, expected", json_test_cases, ids=[_[1] for _ in json_test_cases]) - def test_correct(self, bot_helper, num, url, content_type, fixture, expected): - result = bot_helper['xkcd']._xkcd(num) + async def test_correct(self, bot_helper, num, url, content_type, fixture, expected): + result = await bot_helper['xkcd']._xkcd(num) assert result == expected + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") - def test_latest_success(self, bot_helper): + async def test_latest_success(self, bot_helper): # Also test the empty string num, url, content_type, fixture, expected = json_test_cases[0] - assert bot_helper['xkcd']._xkcd("") == expected + assert await bot_helper['xkcd']._xkcd("") == expected + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") - def test_random(self, bot_helper): + async def test_random(self, bot_helper): # !xkcd 221 num, url, content_type, fixture, expected = json_test_cases[1] with patch("random.randint", return_value=1): - assert bot_helper['xkcd']._xkcd("rand") == expected + assert await bot_helper['xkcd']._xkcd("rand") == expected - def test_error(self, bot_helper, responses): + @pytest.mark.asyncio + async def test_error_1(self, bot_helper, aioresponses): num, url, content_type, fixture, _ = json_test_cases[0] # Latest # Test if the comics are unavailable by making the latest return a 404 - responses.add(responses.GET, url, body="404 - Not Found", - content_type="text/html", status=404) + aioresponses.get(url, body="404 - Not Found", + content_type="text/html", status=404) with pytest.raises(bot_helper['xkcd'].XKCDError): - bot_helper['xkcd']._xkcd("") - responses.reset() + await bot_helper['xkcd']._xkcd("") + @pytest.mark.asyncio + async def test_error_2(self, bot_helper, aioresponses): + num, url, content_type, fixture, _ = json_test_cases[0] # Latest # Now override the actual 404 page and the latest "properly" - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) - responses.add(responses.GET, "http://xkcd.com/404/info.0.json", - body="404 - Not Found", content_type="text/html", - status=404) + aioresponses.get(url, body=read_fixture_file(fixture), + content_type=content_type, + repeat=True) + aioresponses.get("http://xkcd.com/404/info.0.json", + body="404 - Not Found", content_type="text/html", + status=404) error_cases = [ "flibble", @@ -139,7 +145,7 @@ def test_error(self, bot_helper, responses): for case in error_cases: with pytest.raises(bot_helper['xkcd'].XKCDError): - bot_helper['xkcd']._xkcd(case) + await bot_helper['xkcd']._xkcd(case) @pytest.mark.bot(config="""\ @@ -148,25 +154,26 @@ def test_error(self, bot_helper, responses): """) class TestXKCDLinkInfoIntegration: @pytest.fixture - def populate_responses(self, bot_helper, responses): - """Populate all data into responses, don't assert that every request is fired.""" - responses.assert_all_requests_are_fired = False + def populate_responses(self, bot_helper, aioresponses): + """Populate all data into responses.""" for num, url, content_type, fixture, expected in json_test_cases: - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) + aioresponses.get(url, body=read_fixture_file(fixture), + content_type=content_type) @pytest.mark.usefixtures("populate_responses") + @pytest.mark.asyncio @pytest.mark.parametrize("num, url, content_type, fixture, expected", json_test_cases, ids=[_[1] for _ in json_test_cases]) - def test_integration(self, bot_helper, num, url, content_type, fixture, expected): + async def test_integration(self, bot_helper, num, url, content_type, fixture, expected): _, title, alt = expected url = 'http://xkcd.com/{}'.format(num) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) assert title in result.text assert alt in result.text @pytest.mark.usefixtures("populate_responses") - def test_integration_error(self, bot_helper): + @pytest.mark.asyncio + async def test_integration_error(self, bot_helper): # Error case - result = bot_helper['linkinfo'].get_link_info("http://xkcd.com/flibble") + result = await bot_helper['linkinfo'].get_link_info("http://xkcd.com/flibble") assert result.is_error diff --git a/csbot/test/test_plugin_youtube.py b/tests/test_plugin_youtube.py similarity index 56% rename from csbot/test/test_plugin_youtube.py rename to tests/test_plugin_youtube.py index 9552fd68..9d4d180c 100644 --- a/csbot/test/test_plugin_youtube.py +++ b/tests/test_plugin_youtube.py @@ -1,12 +1,10 @@ -from unittest.mock import patch -from distutils.version import StrictVersion +import re import pytest import urllib.parse as urlparse -import apiclient -from csbot.test import fixture_file -from csbot.plugins.youtube import Youtube, YoutubeError +from . import read_fixture_file +from csbot.plugins.youtube import YoutubeError #: Tests are (number, url, content-type, status, fixture, expected) @@ -60,58 +58,62 @@ "youtube_flibble.json", None ), -] -# Non-success HttpMock results are broken in older versions of google-api-python-client -if StrictVersion(apiclient.__version__) > StrictVersion("1.4.1"): - json_test_cases += [ - # Invalid API key (400 Bad Request) - ( - "dQw4w9WgXcQ", - 400, - "youtube_invalid_key.json", - YoutubeError - ), - - # Valid API key, but Youtube Data API not enabled (403 Forbidden) - ( - "dQw4w9WgXcQ", - 403, - "youtube_access_not_configured.json", - YoutubeError - ), - ] + # Invalid API key (400 Bad Request) + ( + "dQw4w9WgXcQ", + 400, + "youtube_invalid_key.json", + YoutubeError + ), + + # Valid API key, but Youtube Data API not enabled (403 Forbidden) + ( + "dQw4w9WgXcQ", + 403, + "youtube_access_not_configured.json", + YoutubeError + ), +] @pytest.fixture -def pre_irc_client(): +def pre_irc_client(aioresponses): # Use fixture JSON for API client setup - http = apiclient.http.HttpMock( - fixture_file('google-discovery-youtube-v3.json'), - {'status': '200'}) - with patch.object(Youtube, 'http', wraps=http): - yield + aioresponses.get('https://www.googleapis.com/discovery/v1/apis/youtube/v3/rest', + status=200, content_type='application/json', + body=read_fixture_file('google-discovery-youtube-v3.json'), + repeat=True) @pytest.mark.bot(config="""\ [@bot] plugins = youtube + + [youtube] + api_key = abc """) class TestYoutubePlugin: + @pytest.mark.asyncio @pytest.mark.parametrize("vid_id, status, fixture, expected", json_test_cases) - def test_ids(self, bot_helper, vid_id, status, fixture, expected): - http = apiclient.http.HttpMock(fixture_file(fixture), {'status': status}) - with patch.object(bot_helper['youtube'], 'http', wraps=http): - if expected is YoutubeError: - with pytest.raises(YoutubeError): - bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) - else: - assert bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) == expected + async def test_ids(self, bot_helper, aioresponses, vid_id, status, fixture, expected): + pattern = re.compile(rf'https://www.googleapis.com/youtube/v3/videos\?.*\bid={vid_id}\b.*') + aioresponses.get(pattern, status=status, content_type='application/json', + body=read_fixture_file(fixture)) + + if expected is YoutubeError: + with pytest.raises(YoutubeError): + await bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) + else: + assert await bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) == expected @pytest.mark.bot(config="""\ [@bot] plugins = linkinfo youtube + + [youtube] + api_key = abc """) class TestYoutubeLinkInfoIntegration: @pytest.fixture @@ -125,6 +127,7 @@ def bot_helper(self, bot_helper): }) return bot_helper + @pytest.mark.asyncio @pytest.mark.parametrize("vid_id, status, fixture, response", json_test_cases) @pytest.mark.parametrize("url", [ "https://www.youtube.com/watch?v={}", @@ -133,15 +136,17 @@ def bot_helper(self, bot_helper): "http://www.youtube.com/watch?v={}&feature=youtube_gdata_player", "http://youtu.be/{}", ]) - def test_integration(self, bot_helper, vid_id, status, fixture, response, url): - http = apiclient.http.HttpMock(fixture_file(fixture), {'status': status}) - with patch.object(bot_helper['youtube'], 'http', wraps=http): - url = url.format(vid_id) - result = bot_helper['linkinfo'].get_link_info(url) - if response is None or response is YoutubeError: - assert result.is_error - else: - for key in response: - if key == "link": - continue - assert response[key] in result.text + async def test_integration(self, bot_helper, aioresponses, vid_id, status, fixture, response, url): + pattern = re.compile(rf'https://www.googleapis.com/youtube/v3/videos\?.*\bid={vid_id}\b.*') + aioresponses.get(pattern, status=status, content_type='application/json', + body=read_fixture_file(fixture)) + + url = url.format(vid_id) + result = await bot_helper['linkinfo'].get_link_info(url) + if response is None or response is YoutubeError: + assert result.is_error + else: + for key in response: + if key == "link": + continue + assert response[key] in result.text diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 00000000..3413fa28 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,60 @@ +import asyncio +from unittest import mock + +import pytest + +from csbot import util + + +@pytest.mark.asyncio +async def test_maybe_future_none(): + assert util.maybe_future(None) is None + + +@pytest.mark.asyncio +async def test_maybe_future_non_awaitable(): + on_error = mock.Mock(spec=callable) + assert util.maybe_future("foo", on_error=on_error) is None + assert on_error.mock_calls == [ + mock.call("foo"), + ] + + +@pytest.mark.asyncio +async def test_maybe_future_coroutine(): + async def foo(): + await asyncio.sleep(0) + return "bar" + + future = util.maybe_future(foo()) + assert future is not None + assert not future.done() + await future + assert future.done() + assert future.exception() is None + + +@pytest.mark.asyncio +async def test_maybe_future_result_none(): + result = await util.maybe_future_result(None) + assert result is None + + +@pytest.mark.asyncio +async def test_maybe_future_result_non_awaitable(): + on_error = mock.Mock(spec=callable) + result = await util.maybe_future_result("foo", on_error=on_error) + assert result == "foo" + assert on_error.mock_calls == [ + mock.call("foo"), + ] + + +@pytest.mark.asyncio +async def test_maybe_future_result_coroutine(): + async def foo(): + await asyncio.sleep(0) + return "bar" + + result = await util.maybe_future_result(foo()) + assert result == "bar" diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..789009b1 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = py36,py37 +skipsdist = True + +[testenv] +passenv = TRAVIS TRAVIS_* +deps = + -r requirements.txt + coveralls: coveralls +commands = + python -m pytest {posargs} + # Try to run coveralls, but don't fail if coveralls fails + coveralls: - coveralls