From 473e7e897d721272b57f6d903aa81ff85720a5a9 Mon Sep 17 00:00:00 2001 From: Thorsten Date: Wed, 5 Aug 2026 17:35:58 +0200 Subject: [PATCH] Fix silently missing titles for pages with a late fetch_page() only ever read the first 8 KB of a response, so any page whose <title> sits beyond that (github: byte 24571) produced no title at all -- the log showed "fetching page ..." and then nothing, and the plugin published an empty action. Read in chunks until shows up instead, capped at 512 KB, and skip the body entirely for non-text responses. Also along that path: - decode with errors='replace'; the read boundary can land mid-character - extract_title() passed user_agent=None, and requests treats a None header value as "drop this header", so every non-YouTube fetch went out with no User-Agent at all - the non-text branch returned the tuple (1, content-type), which is truthy, so URLResolver called .strip() on it and raised AttributeError - log a warning on the remaining no-title paths, which were silent - return None instead of Action(msg="") when nothing resolved Co-Authored-By: Claude Opus 5 --- src/distbot/common/utils.py | 30 ++++++++++++++++++++++++------ src/distbot/plugins/url.py | 3 +++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/distbot/common/utils.py b/src/distbot/common/utils.py index 6ca819a..94bfa3b 100644 --- a/src/distbot/common/utils.py +++ b/src/distbot/common/utils.py @@ -11,6 +11,9 @@ from functools import wraps logger = logging.getLogger(__name__) BUFSIZ = 8192 +# some sites (github, wordpress with fat inline scripts) put way +# beyond the first few KB, so keep reading until it shows up +MAX_FETCH_BYTES = 512 * 1024 USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/108.0.5359.112 Mobile/15E148 Safari/604.1' @@ -74,8 +77,22 @@ def fetch_page(url, user_agent=USER_AGENT): log = logging.getLogger(__name__) log.info('fetching page ' + url) response = requests.get(url, headers={'User-Agent': user_agent}, stream=True, timeout=15) - content = response.raw.read(BUFSIZ, decode_content=True) - return content.decode(response.encoding or 'utf-8'), response.headers + + content_type = response.headers.get('content-type', '') + if content_type and not content_type.startswith('text/'): + # nothing to parse, don't pull the body at all + response.close() + return '', response.headers + + content = b'' + for chunk in response.iter_content(BUFSIZ): + content += chunk + if b'' in content or len(content) >= MAX_FETCH_BYTES: + break + response.close() + + # the last chunk can end mid-character, and encodings get lied about + return content.decode(response.encoding or 'utf-8', errors='replace'), response.headers def extract_title(url): @@ -88,15 +105,14 @@ def extract_title(url): logger.info('extracting title from ' + url) try: - user_agent = None + user_agent = USER_AGENT # sick bastards, writing title with JS if "youtube.com" in url or "youtu.be" in url: user_agent = "curl" (html_text, headers) = fetch_page(url, user_agent) except URLError as e: - return None - except UnicodeDecodeError: + logger.warning('URLError for %s: %s' % (url, str(e))) return None except Exception as e: return 'failed: %s for %s' % (str(e), url) @@ -105,7 +121,8 @@ def extract_title(url): logger.debug('content-type: ' + headers['content-type']) if 'text/' != headers['content-type'][:len('text/')]: - return 1, headers['content-type'] + # not markup, nothing to extract (callers expect a string or None) + return None result = re.match(r'.*?([^<]*?).*?', html_text, re.S | re.M | re.IGNORECASE) if result: @@ -119,6 +136,7 @@ def extract_title(url): expanded_html = match return expanded_html else: + logger.warning('no found in %d bytes of %s' % (len(html_text), url)) return None diff --git a/src/distbot/plugins/url.py b/src/distbot/plugins/url.py index b151af2..fe6a492 100644 --- a/src/distbot/plugins/url.py +++ b/src/distbot/plugins/url.py @@ -120,6 +120,9 @@ class URLResolver(Worker): message = message.replace('\n', '\\n') out.append(message) + if not out: + return None + return Action(msg="\n".join(out)) -- 2.47.3