proxy: smart redirector

app/terraform/proxy/__init__.py

@@ -1,3 +1,4 @@
+import os.path
 from abc import abstractmethod
 from collections import defaultdict
 import datetime
@@ -13,6 +14,7 @@ from app import app
 from app.extensions import db
 from app.models.base import Group
 from app.models.mirrors import Proxy, Origin, SmartProxy
+from app.terraform.proxy.lib import all_cdn_prefixes
 from app.terraform.terraform import TerraformAutomation
 
 
@@ -151,11 +153,15 @@ class ProxyAutomation(TerraformAutomation):
             proxies=Proxy.query.filter(
                 Proxy.provider == self.provider, Proxy.destroyed.is_(None)).all(), subgroups=self.get_subgroups(),
             global_namespace=app.config['GLOBAL_NAMESPACE'], bypass_token=app.config['BYPASS_TOKEN'],
+            terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'),
             **{k: app.config[k.upper()] for k in self.template_parameters})
         if self.smart_proxies:
             for group in groups:
                 self.sp_config(group)
 
+    def sp_trusted_prefixes(self):
+        return "\n".join([f"geoip2_proxy {p};" for p in all_cdn_prefixes()])
+
     def sp_config(self, group: Group) -> None:
         group_origins: List[Origin] = Origin.query.filter(
             Origin.group_id == group.id,
@@ -163,17 +169,32 @@ class ProxyAutomation(TerraformAutomation):
             Origin.smart.is_(True)
         ).all()
         self.tmpl_write(f"smart_proxy.{group.id}.conf", """
+        geoip2 /usr/share/GeoIP/GeoIP2-City.mmdb {
+            auto_reload 5m;
+            $geoip2_metadata_country_build metadata build_epoch;
+            $geoip2_data_country_code default=US country iso_code;
+        }
+        """ + self.sp_trusted_prefixes() + """
+        geoip2_proxy_recursive on;
+          map $geoip2_data_country_code $redirect_country {
+          default yes;
+          """ + "\n".join([f"  {cc} no;" for cc in app.config['CENSORED_COUNTRIES']]) + """
+        }
+
         {% for origin in origins %}
         server {
           listen 443 ssl;
           server_name origin-{{ origin.id }}.{{ provider }}.smart.{{ smart_zone[:-1] }};
+          if ($redirect_country = no) {
+            rewrite ^ https://{{ origin.domain_name }}$request_uri break;
+          }
           location / {
             proxy_set_header Accept-Encoding "";
             proxy_ssl_server_name on;
             proxy_pass https://{{ origin.domain_name }}/;
             subs_filter_types text/html text/css text/xml;
             subs_filter https://{{ origin.domain_name }}/ /;
-            subs_filter "\\\"https://{{ origin.domain_name }}\\\"" /;
+            subs_filter "([^:]|)\\\"https://{{ origin.domain_name }}\\\"" \\1\\\"/\\\";
             {%- for asset_origin in origin.group.origins | selectattr("assets") -%}
             {%- for asset_proxy in asset_origin.proxies | selectattr("provider", "equalto", provider) | selectattr("deprecated", "none") | selectattr("destroyed", "none") -%}
             {%- if loop.first %}

app/terraform/proxy/cloudfront.py

@@ -18,7 +18,9 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
         "rfc2136_nameserver",
         "rfc2136_tsig_key",
         "rfc2136_tsig_secret",
-        "smart_zone"
+        "smart_zone",
+        "maxmind_account_id",
+        "maxmind_license_key"
     ]
 
     template = """
@@ -53,7 +55,7 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
         key_name      = local.rfc2136_tsig_key
         key_secret    = local.rfc2136_tsig_secret
         key_algorithm = "hmac-sha512"
-        timeout       = "10s"
+        timeout       = "60s"
       }
     }
 
@@ -100,6 +102,8 @@ class ProxyCloudfrontAutomation(ProxyAutomation):
       rfc2136_nameserver = local.rfc2136_nameserver
       rfc2136_tsig_key = local.rfc2136_tsig_key
       rfc2136_tsig_secret = local.rfc2136_tsig_secret
+      maxmind_account_id = "{{ maxmind_account_id }}"
+      maxmind_license_key = "{{ maxmind_license_key }}"
     }
 
     resource "aws_s3_object" "smart_config_{{ group.id }}" {

app/terraform/proxy/lib.py

@@ -0,0 +1,73 @@
+import ipaddress
+from typing import List, Dict, Any, Optional, Union, Set, Iterable
+
+import requests
+
+
+class CDNRange:
+    ipv4_ranges: List[ipaddress.IPv4Network]
+    ipv6_ranges: List[ipaddress.IPv6Network]
+
+    def __init__(self) -> None:
+        self.ipv4_ranges = list()
+        self.ipv6_ranges = list()
+
+
+class AWS(CDNRange):
+    def __init__(self, *, data: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__()
+        if data is None:
+            data = requests.get("https://ip-ranges.amazonaws.com/ip-ranges.json").json()
+        self.ipv4_ranges.extend([ipaddress.ip_network(p["ip_prefix"]) for p in data["prefixes"]])  # type: ignore[misc]
+        self.ipv6_ranges.extend([ipaddress.ip_network(p["ipv6_prefix"]) for p in data["ipv6_prefixes"]])  # type: ignore[misc]
+
+
+class AWSCloudFront(CDNRange):
+    def __init__(self, *, data: Optional[Dict[str, List[str]]] = None) -> None:
+        super().__init__()
+        if data is None:
+            data = requests.get("https://d7uri8nf7uskq.cloudfront.net/tools/list-cloudfront-ips").json()
+        for key in data.keys():
+            for item in data[key]:
+                network = ipaddress.ip_network(item)
+                if isinstance(network, ipaddress.IPv4Network):
+                    self.ipv4_ranges.append(network)
+                else:
+                    self.ipv6_ranges.append(network)
+
+
+class AzureFrontDoorBackend(CDNRange):
+    def __init__(self, *, data: Optional[List[Dict[str, Any]]] = None) -> None:
+        super().__init__()
+        if data is None:
+            data = requests.get(
+                "https://azureipranges.azurewebsites.net/getPrefixes/Public/AzureFrontDoor.Backend").json()
+        for item in data[0]["addressPrefixes"]:
+            range = ipaddress.ip_network(item)
+            if isinstance(range, ipaddress.IPv4Network):
+                self.ipv4_ranges.append(range)
+            else:
+                self.ipv6_ranges.append(range)
+
+
+class Fastly(CDNRange):
+    def __init__(self, *, data: Optional[Dict[str, List[str]]] = None) -> None:
+        super().__init__()
+        if data is None:
+            data = requests.get("https://api.fastly.com/public-ip-list").json()
+        self.ipv4_ranges.extend([ipaddress.ip_network(p) for p in data["addresses"]])  # type: ignore[misc]
+        self.ipv6_ranges.extend([ipaddress.ip_network(p) for p in data["ipv6_addresses"]])  # type: ignore[misc]
+
+
+def all_cdn_prefixes() -> Iterable[str]:
+    prefixes: Set[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = set()
+    aws = AWS()
+    prefixes.update(aws.ipv4_ranges)
+    prefixes.update(aws.ipv6_ranges)
+    azure = AzureFrontDoorBackend()
+    prefixes.update(azure.ipv4_ranges)
+    prefixes.update(azure.ipv6_ranges)
+    fastly = Fastly()
+    prefixes.update(fastly.ipv4_ranges)
+    prefixes.update(fastly.ipv6_ranges)
+    return [str(p) for p in prefixes]