fetch_page() only ever read the first 8 KB of a response, so any page
whose <title> sits beyond that (github: byte 24571) produced no title at
all -- the log showed "fetching page ..." and then nothing, and the
plugin published an empty action. Read in chunks until </title> shows
up instead, capped at 512 KB, and skip the body entirely for non-text
responses.
Also along that path:
- decode with errors='replace'; the read boundary can land mid-character
- extract_title() passed user_agent=None, and requests treats a None
header value as "drop this header", so every non-YouTube fetch went
out with no User-Agent at all
- the non-text branch returned the tuple (1, content-type), which is
truthy, so URLResolver called .strip() on it and raised AttributeError
- log a warning on the remaining no-title paths, which were silent
- return None instead of Action(msg="") when nothing resolved
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
logger = logging.getLogger(__name__)
BUFSIZ = 8192
logger = logging.getLogger(__name__)
BUFSIZ = 8192
+# some sites (github, wordpress with fat inline <head> scripts) put <title> way
+# beyond the first few KB, so keep reading until it shows up
+MAX_FETCH_BYTES = 512 * 1024
USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/108.0.5359.112 Mobile/15E148 Safari/604.1'
USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/108.0.5359.112 Mobile/15E148 Safari/604.1'
log = logging.getLogger(__name__)
log.info('fetching page ' + url)
response = requests.get(url, headers={'User-Agent': user_agent}, stream=True, timeout=15)
log = logging.getLogger(__name__)
log.info('fetching page ' + url)
response = requests.get(url, headers={'User-Agent': user_agent}, stream=True, timeout=15)
- content = response.raw.read(BUFSIZ, decode_content=True)
- return content.decode(response.encoding or 'utf-8'), response.headers
+
+ content_type = response.headers.get('content-type', '')
+ if content_type and not content_type.startswith('text/'):
+ # nothing to parse, don't pull the body at all
+ response.close()
+ return '', response.headers
+
+ content = b''
+ for chunk in response.iter_content(BUFSIZ):
+ content += chunk
+ if b'</title>' in content or len(content) >= MAX_FETCH_BYTES:
+ break
+ response.close()
+
+ # the last chunk can end mid-character, and encodings get lied about
+ return content.decode(response.encoding or 'utf-8', errors='replace'), response.headers
logger.info('extracting title from ' + url)
try:
logger.info('extracting title from ' + url)
try:
+ user_agent = USER_AGENT
# sick bastards, writing title with JS
if "youtube.com" in url or "youtu.be" in url:
user_agent = "curl"
(html_text, headers) = fetch_page(url, user_agent)
except URLError as e:
# sick bastards, writing title with JS
if "youtube.com" in url or "youtu.be" in url:
user_agent = "curl"
(html_text, headers) = fetch_page(url, user_agent)
except URLError as e:
- return None
- except UnicodeDecodeError:
+ logger.warning('URLError for %s: %s' % (url, str(e)))
return None
except Exception as e:
return 'failed: %s for %s' % (str(e), url)
return None
except Exception as e:
return 'failed: %s for %s' % (str(e), url)
logger.debug('content-type: ' + headers['content-type'])
if 'text/' != headers['content-type'][:len('text/')]:
logger.debug('content-type: ' + headers['content-type'])
if 'text/' != headers['content-type'][:len('text/')]:
- return 1, headers['content-type']
+ # not markup, nothing to extract (callers expect a string or None)
+ return None
result = re.match(r'.*?<title.*?>([^<]*?)</title>.*?', html_text, re.S | re.M | re.IGNORECASE)
if result:
result = re.match(r'.*?<title.*?>([^<]*?)</title>.*?', html_text, re.S | re.M | re.IGNORECASE)
if result:
expanded_html = match
return expanded_html
else:
expanded_html = match
return expanded_html
else:
+ logger.warning('no <title> found in %d bytes of %s' % (len(html_text), url))
message = message.replace('\n', '\\n')
out.append(message)
message = message.replace('\n', '\\n')
out.append(message)
+ if not out:
+ return None
+
return Action(msg="\n".join(out))
return Action(msg="\n".join(out))
projects / urlbot-v3.git / commitdiff