From 6c88a57ad528a063dedcc44dfa440f7d29e1a3c4 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Fri, 12 Aug 2022 11:55:14 +0100 Subject: [PATCH] proxy: smart redirector --- app/terraform/proxy/__init__.py | 23 +++++++++- app/terraform/proxy/cloudfront.py | 8 +++- app/terraform/proxy/lib.py | 73 +++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 app/terraform/proxy/lib.py diff --git a/app/terraform/proxy/__init__.py b/app/terraform/proxy/__init__.py index 32875a6..a188a2c 100644 --- a/app/terraform/proxy/__init__.py +++ b/app/terraform/proxy/__init__.py @@ -1,3 +1,4 @@ +import os.path from abc import abstractmethod from collections import defaultdict import datetime @@ -13,6 +14,7 @@ from app import app from app.extensions import db from app.models.base import Group from app.models.mirrors import Proxy, Origin, SmartProxy +from app.terraform.proxy.lib import all_cdn_prefixes from app.terraform.terraform import TerraformAutomation @@ -151,11 +153,15 @@ class ProxyAutomation(TerraformAutomation): proxies=Proxy.query.filter( Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(), global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'], + terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'), **{k: app.config[k.upper()] for k in self.template_parameters}) if self.smart_proxies: for group in groups: self.sp_config(group) + def sp_trusted_prefixes(self): + return "\n".join([f"geoip2_proxy {p};" for p in all_cdn_prefixes()]) + def sp_config(self, group: Group) -> None: group_origins: List[Origin] = Origin.query.filter( Origin.group_id == group.id, @@ -163,17 +169,32 @@ class ProxyAutomation(TerraformAutomation): Origin.smart.is_(True) ).all() self.tmpl_write(f"smart_proxy.{group.id}.conf", """ + geoip2 /usr/share/GeoIP/GeoIP2-City.mmdb { + auto_reload 5m; + $geoip2_metadata_country_build metadata build_epoch; + $geoip2_data_country_code default=US country iso_code; + } + """ + self.sp_trusted_prefixes() + """ + geoip2_proxy_recursive on; + map $geoip2_data_country_code $redirect_country { + default yes; + """ + "\n".join([f" {cc} no;" for cc in app.config['CENSORED_COUNTRIES']]) + """ + } + {% for origin in origins %} server { listen 443 ssl; server_name origin-{{ origin.id }}.{{ provider }}.smart.{{ smart_zone[:-1] }}; + if ($redirect_country = no) { + rewrite ^ https://{{ origin.domain_name }}$request_uri break; + } location / { proxy_set_header Accept-Encoding ""; proxy_ssl_server_name on; proxy_pass https://{{ origin.domain_name }}/; subs_filter_types text/html text/css text/xml; subs_filter https://{{ origin.domain_name }}/ /; - subs_filter "\\\"https://{{ origin.domain_name }}\\\"" /; + subs_filter "([^:]|)\\\"https://{{ origin.domain_name }}\\\"" \\1\\\"/\\\"; {%- for asset_origin in origin.group.origins | selectattr("assets") -%} {%- for asset_proxy in asset_origin.proxies | selectattr("provider", "equalto", provider) | selectattr("deprecated", "none") | selectattr("destroyed", "none") -%} {%- if loop.first %} diff --git a/app/terraform/proxy/cloudfront.py b/app/terraform/proxy/cloudfront.py index 8dbf031..739d04d 100644 --- a/app/terraform/proxy/cloudfront.py +++ b/app/terraform/proxy/cloudfront.py @@ -18,7 +18,9 @@ class ProxyCloudfrontAutomation(ProxyAutomation): "rfc2136_nameserver", "rfc2136_tsig_key", "rfc2136_tsig_secret", - "smart_zone" + "smart_zone", + "maxmind_account_id", + "maxmind_license_key" ] template = """ @@ -53,7 +55,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation): key_name = local.rfc2136_tsig_key key_secret = local.rfc2136_tsig_secret key_algorithm = "hmac-sha512" - timeout = "10s" + timeout = "60s" } } @@ -100,6 +102,8 @@ class ProxyCloudfrontAutomation(ProxyAutomation): rfc2136_nameserver = local.rfc2136_nameserver rfc2136_tsig_key = local.rfc2136_tsig_key rfc2136_tsig_secret = local.rfc2136_tsig_secret + maxmind_account_id = "{{ maxmind_account_id }}" + maxmind_license_key = "{{ maxmind_license_key }}" } resource "aws_s3_object" "smart_config_{{ group.id }}" { diff --git a/app/terraform/proxy/lib.py b/app/terraform/proxy/lib.py new file mode 100644 index 0000000..b155f65 --- /dev/null +++ b/app/terraform/proxy/lib.py @@ -0,0 +1,73 @@ +import ipaddress +from typing import List, Dict, Any, Optional, Union, Set, Iterable + +import requests + + +class CDNRange: + ipv4_ranges: List[ipaddress.IPv4Network] + ipv6_ranges: List[ipaddress.IPv6Network] + + def __init__(self) -> None: + self.ipv4_ranges = list() + self.ipv6_ranges = list() + + +class AWS(CDNRange): + def __init__(self, *, data: Optional[Dict[str, Any]] = None) -> None: + super().__init__() + if data is None: + data = requests.get("https://ip-ranges.amazonaws.com/ip-ranges.json").json() + self.ipv4_ranges.extend([ipaddress.ip_network(p["ip_prefix"]) for p in data["prefixes"]]) # type: ignore[misc] + self.ipv6_ranges.extend([ipaddress.ip_network(p["ipv6_prefix"]) for p in data["ipv6_prefixes"]]) # type: ignore[misc] + + +class AWSCloudFront(CDNRange): + def __init__(self, *, data: Optional[Dict[str, List[str]]] = None) -> None: + super().__init__() + if data is None: + data = requests.get("https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips").json() + for key in data.keys(): + for item in data[key]: + network = ipaddress.ip_network(item) + if isinstance(network, ipaddress.IPv4Network): + self.ipv4_ranges.append(network) + else: + self.ipv6_ranges.append(network) + + +class AzureFrontDoorBackend(CDNRange): + def __init__(self, *, data: Optional[List[Dict[str, Any]]] = None) -> None: + super().__init__() + if data is None: + data = requests.get( + "https://azureipranges.azurewebsites.net/getPrefixes/Public/AzureFrontDoor.Backend").json() + for item in data[0]["addressPrefixes"]: + range = ipaddress.ip_network(item) + if isinstance(range, ipaddress.IPv4Network): + self.ipv4_ranges.append(range) + else: + self.ipv6_ranges.append(range) + + +class Fastly(CDNRange): + def __init__(self, *, data: Optional[Dict[str, List[str]]] = None) -> None: + super().__init__() + if data is None: + data = requests.get("https://api.fastly.com/public-ip-list").json() + self.ipv4_ranges.extend([ipaddress.ip_network(p) for p in data["addresses"]]) # type: ignore[misc] + self.ipv6_ranges.extend([ipaddress.ip_network(p) for p in data["ipv6_addresses"]]) # type: ignore[misc] + + +def all_cdn_prefixes() -> Iterable[str]: + prefixes: Set[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = set() + aws = AWS() + prefixes.update(aws.ipv4_ranges) + prefixes.update(aws.ipv6_ranges) + azure = AzureFrontDoorBackend() + prefixes.update(azure.ipv4_ranges) + prefixes.update(azure.ipv6_ranges) + fastly = Fastly() + prefixes.update(fastly.ipv4_ranges) + prefixes.update(fastly.ipv6_ranges) + return [str(p) for p in prefixes]