From: Thorsten <mail@aero2k.de>
Date: Tue, 25 Feb 2020 18:53:55 +0000 (+0100)
Subject: fix searx after stats page has moved
X-Git-Url: https://git.aero2k.de/?a=commitdiff_plain;h=918b0bd7fc3c2261d0a703a6b8614b3db596c0a2;p=urlbot-v3.git

fix searx after stats page has moved
---

diff --git a/distbot/plugins/searx.py b/distbot/plugins/searx.py
index 3c1ca1a..90b3670 100644
--- a/distbot/plugins/searx.py
+++ b/distbot/plugins/searx.py
@@ -14,7 +14,6 @@ from distbot.common.utils import retry
 
 logger = logging.getLogger(__name__)
 
-
 if not hasattr(json, 'JSONDecodeError'):
     json.JSONDecodeError = ValueError
 
@@ -37,19 +36,19 @@ class Searx(Worker):
 
     def __init__(self, actionqueue):
         super().__init__(actionqueue)
-        self.search_list = self.fetch_all_searx_engines()
+        try:
+            self.search_list = self.fetch_all_searx_engines()
+        except RuntimeError as e:
+            # will eventually fix itself
+            logger.exception(e)
+            pass
 
     @staticmethod
     def fetch_all_searx_engines():
-        # response = requests.get("http://stats.searx.oe5tpo.com")
-        response = requests.get("https://stats.searx.xyz")
+        response = requests.get("https://searx.space/data/instances.json")
         response.raise_for_status()
-        tree = etree.XML(
-            response.content,
-            parser=html.HTMLParser()
-        )
-        searxes = [str(x) for x in tree.xpath('//span[text()[contains(.,"200 - OK")]]/../..//a/text()') if
-                   str(x).startswith("http")]
+        searxes = [url for (url, details) in response.json().get("instances").items() if
+                   details.get("error") is None and details.get("network_type") != "tor"]
         logger.info("Registered {} searxes".format(len(searxes)))
         if not searxes:
             raise RuntimeError("not a single searx discovered... " + str(response.content))