happy new year, I guess

diff --git a/bin/urlbotd-chat b/bin/urlbotd-chat
new file mode 100755 (executable)
index 0000000..01fd4cc
--- /dev/null
+++ b/bin/urlbotd-chat
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from distbot.bot import bot
+
+bot.run()

diff --git a/bin/urlbotd-worker b/bin/urlbotd-worker
new file mode 100755 (executable)
index 0000000..0c1130c
--- /dev/null
+++ b/bin/urlbotd-worker
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from distbot.minijobber import run
+
+run.run()

diff --git a/deploy/README b/deploy/README
new file mode 100644 (file)
index 0000000..979315e
--- /dev/null
+++ b/deploy/README
@@ -0,0 +1,30 @@
+To use the playbook, create a yaml file credentials.yml
+with the following content (you can use vault to encrypt):
+
+jid: yourjabber@id.to.use
+password: yourpasswordforthisjabber
+rooms:
+   - debianforum.de@chat.debianforum.de # your channel
+   - spielwiese@chat.debianforum.de # bot playground
+
+bot_nickname: T800 # your bots nickname
+bot_owner: MASTER # your nickname (for info and admin stuff)
+bot_owner_email: ... # same
+
+amqp_uri: "amqp://user:pass@host:5672/%2F" # url to mq system, rabbitmq has been tested
+
+Further, you need a hosts-file with the following content:
+
+[bots]
+yourserverip
+# or alternatively,
+derpy_name ansible_host=yourserverip
+
+
+There is deploy.sh which I created so I have a single file to
+deploy my stuff - it uses a virtualenv with ansible and
+ some vault file declaration.
+
+Or just put the contents of the repository somewhere, pip install . and run the files in /bin.
+
+In any case, rabbitmq (or comparable, I guess) is required. Setup TBD.

diff --git a/deploy/deploy.sh b/deploy/deploy.sh
new file mode 100755 (executable)
index 0000000..d13c998
--- /dev/null
+++ b/deploy/deploy.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source ~/.virtualenvs/ansible/bin/activate
+
+export ANSIBLE_VAULT_PASSWORD_FILE=vault.sh
+
+ansible-playbook -i hosts deploy.yml -D

diff --git a/deploy/deploy.yml b/deploy/deploy.yml
new file mode 100644 (file)
index 0000000..2ba3173
--- /dev/null
+++ b/deploy/deploy.yml
@@ -0,0 +1,24 @@
+---
+- hosts: bots
+  remote_user: root
+  tasks:
+  - name: create user for bot
+    user:
+      # default name
+      name: jabberbot
+      comment: "Account for the urlbot"
+      shell: /bin/bash
+  - name: local user can log in with ssh key
+    authorized_key: user=jabberbot key="{{ lookup('file', '~/.ssh/id_rsa.pub') }}"
+  - name: ensure venv module is installed
+    apt:
+      name: python3-venv
+      state: present
+
+- hosts: bots
+  vars:
+    botuser: jabberbot
+  remote_user: "{{ botuser }}"
+  roles:
+    - urlbot
+

diff --git a/deploy/requirements-deploy.yml b/deploy/requirements-deploy.yml
new file mode 100644 (file)
index 0000000..7547399
--- /dev/null
+++ b/deploy/requirements-deploy.yml
@@ -0,0 +1,2 @@
+ansible
+markupsafe

diff --git a/deploy/roles/urlbot/defaults/main.yml b/deploy/roles/urlbot/defaults/main.yml
new file mode 100644 (file)
index 0000000..ecd8315
--- /dev/null
+++ b/deploy/roles/urlbot/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+# local user to be used
+botuser: jabberbot
\ No newline at end of file

diff --git a/deploy/roles/urlbot/handler/main.yml b/deploy/roles/urlbot/handler/main.yml
new file mode 100644 (file)
index 0000000..b303201
--- /dev/null
+++ b/deploy/roles/urlbot/handler/main.yml
@@ -0,0 +1,12 @@
+---
+name: restart chatbot
+systemd:
+  scope: user
+  name: urlbot-chat.service
+  state: restarted
+
+name: restart worker
+systemd:
+  scope: user
+  name: urlbot-worker.service
+  state: restarted

diff --git a/deploy/roles/urlbot/tasks/main.yml b/deploy/roles/urlbot/tasks/main.yml
new file mode 100644 (file)
index 0000000..88d97be
--- /dev/null
+++ b/deploy/roles/urlbot/tasks/main.yml
@@ -0,0 +1,77 @@
+---
+
+- name: setup virtualenv for the chatbot
+  shell:
+    cmd: "python3 -m venv {{ venv_chatbot }}"
+    creates: "{{ venv_chatbot }}"
+- name: setup virtualenv for the workers
+  shell:
+    cmd: "python3 -m venv {{ venv_worker }}"
+    creates: "{{ venv_worker }}"
+
+- name: clone repository
+  git:
+    repo: "{{botrepo}}"
+    dest: ~/urlbot-v3
+    force: yes
+    update: yes
+  register: source_code
+  # TODO: distinction of changes
+
+- name: install bot (xmpp)
+  shell: "{{ venv_chatbot }}/bin/pip install -i {{pypi_mirror}} .[chatbot]"
+  args:
+    chdir: ~/urlbot-v3
+
+- name: install bot (worker)
+  shell: "{{ venv_worker }}/bin/pip install -i {{pypi_mirror}} .[worker]"
+  args:
+    chdir: ~/urlbot-v3
+
+- name: set configuration
+  lineinfile: dest=~/urlbot/local_config.ini create=yes line="{{item.key}} = {{item.value}}" regexp="^{{item.key}}.="
+  with_items:
+    - key: "jid"
+      value: "{{jid}}"
+    - key: "password"
+      value: "{{password}}"
+    - key: "rooms"
+      value: "{{rooms | join(', ')}}"
+    - key: "src-url"
+      value: "{{botrepo}}"
+    - key: "bot_nickname"
+      value: "{{bot_nickname}}"
+    - key: "bot_owner"
+      value: "{{bot_owner}}"
+    - key: "bot_owner_email"
+      value: "{{bot_owner_email}}"
+    - key: "amqp_uri"
+      value: "{{ amqp_uri }}"
+    # TODO: detectlanguage_api_key, sudoers and giphy_key
+    - key: "giphy_key"
+      value: "{{giphy_key}}"
+  tags: [render_config]
+  register: urlbot_config
+
+- name: create directory for systemd unit files
+  shell: mkdir -p ~/.config/systemd/user/ creates=~/.config/systemd/user/
+
+- name: unitfile
+  template:
+    src: "urlbot-{{ item }}.service"
+    dest: "~/.config/systemd/user/urlbot-{{ item }}.service"
+  register: unitfile
+  with_items:
+    - chat
+    - worker
+
+- systemd:
+    name: "urlbot-{{ item }}.service"
+    daemon-reload: true
+    scope: user
+    enabled: true
+    state: started
+  with_items:
+    - chat
+    - worker
+

diff --git a/deploy/roles/urlbot/templates/urlbot-chat.service b/deploy/roles/urlbot/templates/urlbot-chat.service
new file mode 100644 (file)
index 0000000..eb037de
--- /dev/null
+++ b/deploy/roles/urlbot/templates/urlbot-chat.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=jabber bot entertaining and supporting activity on jabber MUCs
+
+[Service]
+ExecStart={{ venv_chatbot }}/bin/urlbotd-chat
+WorkingDirectory=/home/{{ botuser }}/urlbot-v3/
+StandardOutput=journal+console
+StandardError=journal+console
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

diff --git a/deploy/roles/urlbot/templates/urlbot-worker.service b/deploy/roles/urlbot/templates/urlbot-worker.service
new file mode 100644 (file)
index 0000000..5f56b14
--- /dev/null
+++ b/deploy/roles/urlbot/templates/urlbot-worker.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=jabber bot entertaining and supporting activity on jabber MUCs
+
+[Service]
+ExecStart={{ venv_chatbot }}/bin/urlbotd-worker
+WorkingDirectory=/home/{{ botuser }}/urlbot-v3/
+StandardOutput=journal+console
+StandardError=journal+console
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

diff --git a/deploy/roles/urlbot/vars/main.yml b/deploy/roles/urlbot/vars/main.yml
new file mode 100644 (file)
index 0000000..5893611
--- /dev/null
+++ b/deploy/roles/urlbot/vars/main.yml
@@ -0,0 +1,5 @@
+---
+botrepo: http://aero2k.de/t/repos/urlbot-v3.git
+pypi_mirror: https://pypi.fcio.net/simple/
+venv_chatbot: "/home/{{ botuser }}/.virtualenvs/urlbot_xmpp"
+venv_worker: "/home/{{ botuser }}/.virtualenvs/urlbot_worker"
\ No newline at end of file

diff --git a/deploy/vault.sh b/deploy/vault.sh
new file mode 100755 (executable)
index 0000000..fa20d7b
--- /dev/null
+++ b/deploy/vault.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# using pass - the best passwordmanager.org on the world
+
+pass show infra

diff --git a/setup.py b/setup.py
index 45fdeaa24ae45ca3e82aeb95ec54b1f1a61bc917..696551e38ac0fa619e6b851855a35ce9018d30ef 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -4,15 +4,17 @@ setup(
     name="distbot",
     version=0.1,
     description="the distributed bot",
-    install_requires=["sleekxmpp", "pika==0.12.0", "flask", 'configobj', 'requests', 'lxml'],
+    install_requires=["pika==0.12.0", 'configobj', 'requests'],
     test_requires=["pytest"],
     extras_require={
         'test': ["pytest"],
+        'chatbot': ["sleekxmpp", "pyopenssl"],
+        'worker': ["lxml"],
     },
-    entry_points={
-        'console_scripts': [
-        ]
-    },
+    scripts=[
+        'bin/urlbotd-chat',
+        'bin/urlbotd-worker',
+    ],
     packages=find_packages(exclude=['Readme.md', 'tests', 'doc']),
 
 )

diff --git a/tests/test_urlresolver.py b/tests/test_urlresolver.py
new file mode 100644 (file)
index 0000000..357db32
--- /dev/null
+++ b/tests/test_urlresolver.py
@@ -0,0 +1,38 @@
+import pytest
+import requests
+
+from distbot.plugins import url as url_plugin
+
+
+# TODO: mocked results
+# @pytest.fixture(params=[
+#     ("http://"),
+# ])
+# def url(request):
+#     url, title_regex = request.params
+
+# smoketest
+
+
+@pytest.mark.parametrize(
+    argnames='url, title, problem',
+    argvalues=[
+        ('http://debianforum.de', "debianforum.de - Startseite", None),
+        ('http://aero2k.de', '403 Forbidden', None),
+        ('http://debianforum.de:12345/', None, requests.exceptions.ConnectionError),
+        ('ftp://ftp.de.debian.org', None, requests.exceptions.InvalidSchema),
+        ('https://marc.info/', "MARC: Mailing list ARChives", None),
+    ]
+)
+def test_smoke(url, title, problem):
+    def run_test():
+        response = requests.get(url)
+        plugin_title = url_plugin.extract_title(url)
+        assert "<title>{}</title>".format(plugin_title) in response.text
+        assert plugin_title == title
+
+    if problem:
+        with pytest.raises(problem):
+            run_test()
+    else:
+        run_test()