More url checking

This commit is contained in:
Thomas Sileo
2018-05-26 00:03:30 +02:00
parent 06f4f824d8
commit a3267971e8
3 changed files with 10 additions and 1 deletions

View File

@@ -5,7 +5,7 @@ import opengraph
import requests
from bs4 import BeautifulSoup
from .urlutils import is_url_valid
from .urlutils import is_url_valid, check_url
def links_from_note(note):
@@ -38,6 +38,7 @@ def fetch_og_metadata(user_agent, col, remote_id):
# FIXME(tsileo): set the user agent by giving HTML directly to OpenGraph
htmls = []
for l in links:
check_url(l)
r = requests.get(l, headers={'User-Agent': user_agent})
r.raise_for_status()
htmls.append(r.text)