logger = logging.getLogger(__name__)
BUFSIZ = 8192
+# some sites (github, wordpress with fat inline <head> scripts) put <title> way
+# beyond the first few KB, so keep reading until it shows up
+MAX_FETCH_BYTES = 512 * 1024
USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/108.0.5359.112 Mobile/15E148 Safari/604.1'
log = logging.getLogger(__name__)
log.info('fetching page ' + url)
response = requests.get(url, headers={'User-Agent': user_agent}, stream=True, timeout=15)
- content = response.raw.read(BUFSIZ, decode_content=True)
- return content.decode(response.encoding or 'utf-8'), response.headers
+
+ content_type = response.headers.get('content-type', '')
+ if content_type and not content_type.startswith('text/'):
+ # nothing to parse, don't pull the body at all
+ response.close()
+ return '', response.headers
+
+ content = b''
+ for chunk in response.iter_content(BUFSIZ):
+ content += chunk
+ if b'</title>' in content or len(content) >= MAX_FETCH_BYTES:
+ break
+ response.close()
+
+ # the last chunk can end mid-character, and encodings get lied about
+ return content.decode(response.encoding or 'utf-8', errors='replace'), response.headers
def extract_title(url):
logger.info('extracting title from ' + url)
try:
- user_agent = None
+ user_agent = USER_AGENT
# sick bastards, writing title with JS
if "youtube.com" in url or "youtu.be" in url:
user_agent = "curl"
(html_text, headers) = fetch_page(url, user_agent)
except URLError as e:
- return None
- except UnicodeDecodeError:
+ logger.warning('URLError for %s: %s' % (url, str(e)))
return None
except Exception as e:
return 'failed: %s for %s' % (str(e), url)
logger.debug('content-type: ' + headers['content-type'])
if 'text/' != headers['content-type'][:len('text/')]:
- return 1, headers['content-type']
+ # not markup, nothing to extract (callers expect a string or None)
+ return None
result = re.match(r'.*?<title.*?>([^<]*?)</title>.*?', html_text, re.S | re.M | re.IGNORECASE)
if result:
expanded_html = match
return expanded_html
else:
+ logger.warning('no <title> found in %d bytes of %s' % (len(html_text), url))
return None
projects / urlbot-v3.git / commitdiff