From 7c21b2eb4ce6d451353e53b959be14192f79c69d Mon Sep 17 00:00:00 2001 From: Thorsten S Date: Thu, 2 Jan 2020 15:39:26 +0100 Subject: [PATCH] happy new year, I guess --- bin/urlbotd-chat | 5 ++ bin/urlbotd-worker | 5 ++ deploy/README | 30 ++++++++ deploy/deploy.sh | 7 ++ deploy/deploy.yml | 24 ++++++ deploy/requirements-deploy.yml | 2 + deploy/roles/urlbot/defaults/main.yml | 3 + deploy/roles/urlbot/handler/main.yml | 12 +++ deploy/roles/urlbot/tasks/main.yml | 77 +++++++++++++++++++ .../urlbot/templates/urlbot-chat.service | 12 +++ .../urlbot/templates/urlbot-worker.service | 12 +++ deploy/roles/urlbot/vars/main.yml | 5 ++ deploy/vault.sh | 5 ++ setup.py | 12 +-- tests/test_urlresolver.py | 38 +++++++++ 15 files changed, 244 insertions(+), 5 deletions(-) create mode 100755 bin/urlbotd-chat create mode 100755 bin/urlbotd-worker create mode 100644 deploy/README create mode 100755 deploy/deploy.sh create mode 100644 deploy/deploy.yml create mode 100644 deploy/requirements-deploy.yml create mode 100644 deploy/roles/urlbot/defaults/main.yml create mode 100644 deploy/roles/urlbot/handler/main.yml create mode 100644 deploy/roles/urlbot/tasks/main.yml create mode 100644 deploy/roles/urlbot/templates/urlbot-chat.service create mode 100644 deploy/roles/urlbot/templates/urlbot-worker.service create mode 100644 deploy/roles/urlbot/vars/main.yml create mode 100755 deploy/vault.sh create mode 100644 tests/test_urlresolver.py diff --git a/bin/urlbotd-chat b/bin/urlbotd-chat new file mode 100755 index 0000000..01fd4cc --- /dev/null +++ b/bin/urlbotd-chat @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from distbot.bot import bot + +bot.run() diff --git a/bin/urlbotd-worker b/bin/urlbotd-worker new file mode 100755 index 0000000..0c1130c --- /dev/null +++ b/bin/urlbotd-worker @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from distbot.minijobber import run + +run.run() diff --git a/deploy/README b/deploy/README new file mode 100644 index 0000000..979315e --- /dev/null +++ b/deploy/README @@ -0,0 +1,30 @@ +To use the playbook, create a yaml file credentials.yml +with the following content (you can use vault to encrypt): + +jid: yourjabber@id.to.use +password: yourpasswordforthisjabber +rooms: + - debianforum.de@chat.debianforum.de # your channel + - spielwiese@chat.debianforum.de # bot playground + +bot_nickname: T800 # your bots nickname +bot_owner: MASTER # your nickname (for info and admin stuff) +bot_owner_email: ... # same + +amqp_uri: "amqp://user:pass@host:5672/%2F" # url to mq system, rabbitmq has been tested + +Further, you need a hosts-file with the following content: + +[bots] +yourserverip +# or alternatively, +derpy_name ansible_host=yourserverip + + +There is deploy.sh which I created so I have a single file to +deploy my stuff - it uses a virtualenv with ansible and + some vault file declaration. + +Or just put the contents of the repository somewhere, pip install . and run the files in /bin. + +In any case, rabbitmq (or comparable, I guess) is required. Setup TBD. diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 0000000..d13c998 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source ~/.virtualenvs/ansible/bin/activate + +export ANSIBLE_VAULT_PASSWORD_FILE=vault.sh + +ansible-playbook -i hosts deploy.yml -D diff --git a/deploy/deploy.yml b/deploy/deploy.yml new file mode 100644 index 0000000..2ba3173 --- /dev/null +++ b/deploy/deploy.yml @@ -0,0 +1,24 @@ +--- +- hosts: bots + remote_user: root + tasks: + - name: create user for bot + user: + # default name + name: jabberbot + comment: "Account for the urlbot" + shell: /bin/bash + - name: local user can log in with ssh key + authorized_key: user=jabberbot key="{{ lookup('file', '~/.ssh/id_rsa.pub') }}" + - name: ensure venv module is installed + apt: + name: python3-venv + state: present + +- hosts: bots + vars: + botuser: jabberbot + remote_user: "{{ botuser }}" + roles: + - urlbot + diff --git a/deploy/requirements-deploy.yml b/deploy/requirements-deploy.yml new file mode 100644 index 0000000..7547399 --- /dev/null +++ b/deploy/requirements-deploy.yml @@ -0,0 +1,2 @@ +ansible +markupsafe diff --git a/deploy/roles/urlbot/defaults/main.yml b/deploy/roles/urlbot/defaults/main.yml new file mode 100644 index 0000000..ecd8315 --- /dev/null +++ b/deploy/roles/urlbot/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# local user to be used +botuser: jabberbot \ No newline at end of file diff --git a/deploy/roles/urlbot/handler/main.yml b/deploy/roles/urlbot/handler/main.yml new file mode 100644 index 0000000..b303201 --- /dev/null +++ b/deploy/roles/urlbot/handler/main.yml @@ -0,0 +1,12 @@ +--- +name: restart chatbot +systemd: + scope: user + name: urlbot-chat.service + state: restarted + +name: restart worker +systemd: + scope: user + name: urlbot-worker.service + state: restarted diff --git a/deploy/roles/urlbot/tasks/main.yml b/deploy/roles/urlbot/tasks/main.yml new file mode 100644 index 0000000..88d97be --- /dev/null +++ b/deploy/roles/urlbot/tasks/main.yml @@ -0,0 +1,77 @@ +--- + +- name: setup virtualenv for the chatbot + shell: + cmd: "python3 -m venv {{ venv_chatbot }}" + creates: "{{ venv_chatbot }}" +- name: setup virtualenv for the workers + shell: + cmd: "python3 -m venv {{ venv_worker }}" + creates: "{{ venv_worker }}" + +- name: clone repository + git: + repo: "{{botrepo}}" + dest: ~/urlbot-v3 + force: yes + update: yes + register: source_code + # TODO: distinction of changes + +- name: install bot (xmpp) + shell: "{{ venv_chatbot }}/bin/pip install -i {{pypi_mirror}} .[chatbot]" + args: + chdir: ~/urlbot-v3 + +- name: install bot (worker) + shell: "{{ venv_worker }}/bin/pip install -i {{pypi_mirror}} .[worker]" + args: + chdir: ~/urlbot-v3 + +- name: set configuration + lineinfile: dest=~/urlbot/local_config.ini create=yes line="{{item.key}} = {{item.value}}" regexp="^{{item.key}}.=" + with_items: + - key: "jid" + value: "{{jid}}" + - key: "password" + value: "{{password}}" + - key: "rooms" + value: "{{rooms | join(', ')}}" + - key: "src-url" + value: "{{botrepo}}" + - key: "bot_nickname" + value: "{{bot_nickname}}" + - key: "bot_owner" + value: "{{bot_owner}}" + - key: "bot_owner_email" + value: "{{bot_owner_email}}" + - key: "amqp_uri" + value: "{{ amqp_uri }}" + # TODO: detectlanguage_api_key, sudoers and giphy_key + - key: "giphy_key" + value: "{{giphy_key}}" + tags: [render_config] + register: urlbot_config + +- name: create directory for systemd unit files + shell: mkdir -p ~/.config/systemd/user/ creates=~/.config/systemd/user/ + +- name: unitfile + template: + src: "urlbot-{{ item }}.service" + dest: "~/.config/systemd/user/urlbot-{{ item }}.service" + register: unitfile + with_items: + - chat + - worker + +- systemd: + name: "urlbot-{{ item }}.service" + daemon-reload: true + scope: user + enabled: true + state: started + with_items: + - chat + - worker + diff --git a/deploy/roles/urlbot/templates/urlbot-chat.service b/deploy/roles/urlbot/templates/urlbot-chat.service new file mode 100644 index 0000000..eb037de --- /dev/null +++ b/deploy/roles/urlbot/templates/urlbot-chat.service @@ -0,0 +1,12 @@ +[Unit] +Description=jabber bot entertaining and supporting activity on jabber MUCs + +[Service] +ExecStart={{ venv_chatbot }}/bin/urlbotd-chat +WorkingDirectory=/home/{{ botuser }}/urlbot-v3/ +StandardOutput=journal+console +StandardError=journal+console +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/deploy/roles/urlbot/templates/urlbot-worker.service b/deploy/roles/urlbot/templates/urlbot-worker.service new file mode 100644 index 0000000..5f56b14 --- /dev/null +++ b/deploy/roles/urlbot/templates/urlbot-worker.service @@ -0,0 +1,12 @@ +[Unit] +Description=jabber bot entertaining and supporting activity on jabber MUCs + +[Service] +ExecStart={{ venv_chatbot }}/bin/urlbotd-worker +WorkingDirectory=/home/{{ botuser }}/urlbot-v3/ +StandardOutput=journal+console +StandardError=journal+console +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/deploy/roles/urlbot/vars/main.yml b/deploy/roles/urlbot/vars/main.yml new file mode 100644 index 0000000..5893611 --- /dev/null +++ b/deploy/roles/urlbot/vars/main.yml @@ -0,0 +1,5 @@ +--- +botrepo: http://aero2k.de/t/repos/urlbot-v3.git +pypi_mirror: https://pypi.fcio.net/simple/ +venv_chatbot: "/home/{{ botuser }}/.virtualenvs/urlbot_xmpp" +venv_worker: "/home/{{ botuser }}/.virtualenvs/urlbot_worker" \ No newline at end of file diff --git a/deploy/vault.sh b/deploy/vault.sh new file mode 100755 index 0000000..fa20d7b --- /dev/null +++ b/deploy/vault.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# using pass - the best passwordmanager.org on the world + +pass show infra diff --git a/setup.py b/setup.py index 45fdeaa..696551e 100644 --- a/setup.py +++ b/setup.py @@ -4,15 +4,17 @@ setup( name="distbot", version=0.1, description="the distributed bot", - install_requires=["sleekxmpp", "pika==0.12.0", "flask", 'configobj', 'requests', 'lxml'], + install_requires=["pika==0.12.0", 'configobj', 'requests'], test_requires=["pytest"], extras_require={ 'test': ["pytest"], + 'chatbot': ["sleekxmpp", "pyopenssl"], + 'worker': ["lxml"], }, - entry_points={ - 'console_scripts': [ - ] - }, + scripts=[ + 'bin/urlbotd-chat', + 'bin/urlbotd-worker', + ], packages=find_packages(exclude=['Readme.md', 'tests', 'doc']), ) diff --git a/tests/test_urlresolver.py b/tests/test_urlresolver.py new file mode 100644 index 0000000..357db32 --- /dev/null +++ b/tests/test_urlresolver.py @@ -0,0 +1,38 @@ +import pytest +import requests + +from distbot.plugins import url as url_plugin + + +# TODO: mocked results +# @pytest.fixture(params=[ +# ("http://"), +# ]) +# def url(request): +# url, title_regex = request.params + +# smoketest + + +@pytest.mark.parametrize( + argnames='url, title, problem', + argvalues=[ + ('http://debianforum.de', "debianforum.de - Startseite", None), + ('http://aero2k.de', '403 Forbidden', None), + ('http://debianforum.de:12345/', None, requests.exceptions.ConnectionError), + ('ftp://ftp.de.debian.org', None, requests.exceptions.InvalidSchema), + ('https://marc.info/', "MARC: Mailing list ARChives", None), + ] +) +def test_smoke(url, title, problem): + def run_test(): + response = requests.get(url) + plugin_title = url_plugin.extract_title(url) + assert "{}".format(plugin_title) in response.text + assert plugin_title == title + + if problem: + with pytest.raises(problem): + run_test() + else: + run_test() -- 2.39.2