block: fetch and parse are public methods
This commit is contained in:
parent
360c786610
commit
894b26c5f8
2 changed files with 3 additions and 5 deletions
|
@ -1,5 +1,3 @@
|
||||||
from typing import List, Dict
|
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
@ -16,12 +14,12 @@ class BlockExternalAutomation(BlockMirrorAutomation):
|
||||||
|
|
||||||
_content: bytes
|
_content: bytes
|
||||||
|
|
||||||
def _fetch(self) -> None:
|
def fetch(self) -> None:
|
||||||
user_agent = {'User-agent': 'BypassCensorship/1.0'}
|
user_agent = {'User-agent': 'BypassCensorship/1.0'}
|
||||||
page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
|
page = requests.get(app.config['EXTERNAL_CHECK_URL'], headers=user_agent)
|
||||||
self._content = page.content
|
self._content = page.content
|
||||||
|
|
||||||
def _parse(self) -> None:
|
def parse(self) -> None:
|
||||||
soup = BeautifulSoup(self._content, 'html.parser')
|
soup = BeautifulSoup(self._content, 'html.parser')
|
||||||
h2 = soup.find_all('h2') # pylint: disable=invalid-name
|
h2 = soup.find_all('h2') # pylint: disable=invalid-name
|
||||||
div = soup.find_all('div', class_="overflow-auto mb-5")
|
div = soup.find_all('div', class_="overflow-auto mb-5")
|
||||||
|
|
|
@ -50,8 +50,8 @@ class BlockMirrorAutomation(BaseAutomation):
|
||||||
"\n".join([f"* {proxy_domain} ({origin_domain})" for proxy_domain, origin_domain in rotated]))
|
"\n".join([f"* {proxy_domain} ({origin_domain})" for proxy_domain, origin_domain in rotated]))
|
||||||
)
|
)
|
||||||
db.session.add(activity)
|
db.session.add(activity)
|
||||||
db.session.commit()
|
|
||||||
activity.notify()
|
activity.notify()
|
||||||
|
db.session.commit()
|
||||||
return True, ""
|
return True, ""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue