proxy: adds support for fastly provider

app/cli/automate.py

@@ -28,6 +28,7 @@ from app.terraform.list.gitlab import ListGitlabAutomation
 from app.terraform.list.s3 import ListS3Automation
 from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
 from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
+from app.terraform.proxy.fastly import ProxyFastlyAutomation
 
 jobs = {
     x.short_name: x
@@ -50,7 +51,8 @@ jobs = {
         ListGitlabAutomation,
         ListS3Automation,
         ProxyAzureCdnAutomation,
-        ProxyCloudfrontAutomation
+        ProxyCloudfrontAutomation,
+        ProxyFastlyAutomation
     ]
 }
 

app/terraform/proxy/fastly.py

@@ -1,21 +1,22 @@
-# type: ignore
+from typing import Any
-# TODO: This module doesn't work at all
 
-import datetime
+from app.terraform.proxy import ProxyAutomation
-import os
-import random
-import string
 
-import jinja2
-import tldextract
 
-from app import app
+class ProxyFastlyAutomation(ProxyAutomation):
-from app.extensions import db
+    short_name = "proxy_fastly"
-from app.models.base import Group
+    description = "Deploy proxies to Fastly"
-from app.models.mirrors import Origin, Proxy
+    provider = "fastly"
+    subgroup_max = 5
 
-TEMPLATE = """
+    template_parameters = [
-terraform {
+        "aws_access_key",
+        "aws_secret_key",
+        "fastly_api_key"
+    ]
+
+    template = """
+    terraform {
       required_providers {
         aws = {
           version = "~> 4.4.0"
@@ -25,28 +26,28 @@ terraform {
           version = ">= 1.1.1"
         }
       }
-}
+    }
 
-provider "aws" {
+    provider "aws" {
       access_key = "{{ aws_access_key }}"
       secret_key = "{{ aws_secret_key }}"
-  region = "us-east-1"
+      region = "us-east-2"
-}
+    }
 
-provider "fastly" {
+    provider "fastly" {
       api_key = "{{ fastly_api_key }}"
-}
+    }
 
-{% for group in groups %}
+    {% for group in groups %}
-module "label_{{ group.id }}" {
+    module "label_{{ group.id }}" {
       source  = "cloudposse/label/null"
       version = "0.25.0"
       namespace = "bc"
       tenant = "{{ group.group_name }}"
       label_order = ["namespace", "tenant", "name", "attributes"]
-}
+    }
 
-module "log_bucket_{{ group.id }}" {
+    module "log_bucket_{{ group.id }}" {
       source = "cloudposse/s3-log-storage/aws"
       version = "0.28.0"
       context = module.label_{{ group.id }}.context
@@ -56,89 +57,61 @@ module "log_bucket_{{ group.id }}" {
       standard_transition_days = 30
       glacier_transition_days  = 60
       expiration_days          = 90
-}
+    }
 
-{% if group.id == 3 %}
+    {% if group.id == 3 %}
-resource "fastly_service_vcl" "service_{{ group.id }}" {
+    {% for subgroup in subgroups[group.id] %}
-  name = module.label_{{ group.id }}.id
+    resource "fastly_service_vcl" "service_{{ group.id }}_{{ subgroup }}" {
+      name = "${module.label_{{ group.id }}.id}-{{ subgroup }}"
 
       {% for origin in group.origins %}
       {% for proxy in origin.proxies %}
-  {% if proxy.destroyed == None and proxy.provider == "fastly" %}
+      {% if proxy.destroyed == None and proxy.provider == "fastly" and proxy.psg == subgroup %}
       domain {
-    name    = "{{ proxy.slug }}.global.ssl.fastly.com"
+        name    = "{{ proxy.slug }}.global.ssl.fastly.net"
         comment = "Mirror"
       }
-  {% endif %}
-  {% endfor %}
 
       backend {
-    address       = "{{ origin.domain_name }}"
+        address       = "{{ proxy.origin.domain_name }}"
-    name          = "{{ origin.description }}"
+        name          = "{{ proxy.origin.domain_name.replace(".", "_") }}_{{ proxy.id }}"
         port          = 443
-    override_host = "{{ origin.domain_name }}"
+        ssl_hostname  = "{{ proxy.origin.domain_name }}"
+        override_host = "{{ proxy.origin.domain_name }}"
       }
-  {% endfor %}
+      {% endif %}
-}
+      {% endfor %}{# proxy #}
-{% endif %}
+      {% endfor %}{# origin #}
-{% endfor %}
-"""
 
+      snippet {
+        name = "director"
+        content = <<EOV
+            if (req.http.host == "") {
+                error 400 "No host header";
+            }
+            {% for origin in group.origins %}
+            {% for proxy in origin.proxies %}
+            {% if proxy.destroyed == None and proxy.provider == "fastly" and proxy.psg == subgroup %}
+            else if (req.http.host == "{{ proxy.slug }}.global.ssl.fastly.net") {
+              set req.backend = F_{{ origin.domain_name.replace(".", "_") }}_{{ proxy.id }};
+              if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
+                return(pass);
+              }
+              return(lookup);
+            }
+            {% endif %}{% endfor %}{% endfor %}
+            else {
+                error 400 "Unknown host header";
+            }
+            EOV
+        type = "recv"
+      }
+    }
+    {% endfor %}{# subgroup #}
+    {% endif %}{# group.id == 3 #}
 
-def create_missing_proxies():
+    {% endfor %}{# group #}
-    with app.app_context():
+    """
-        origins = Origin.query.filter(Origin.group_id == 3).all()
-        for origin in origins:
-            azure_cdn_proxies = [
-                x for x in origin.proxies
-                if x.provider == "fastly" and x.deprecated is None and x.destroyed is None
-            ]
-            if not azure_cdn_proxies:
-                proxy = Proxy()
-                proxy.origin_id = origin.id
-                proxy.provider = "fastly"
-                # The random usage below is good enough for its purpose: to create a slug that
-                # hasn't been used before.
-                proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
-                    random.choices(string.ascii_lowercase, 12))  # nosec
-                proxy.added = datetime.datetime.utcnow()
-                proxy.updated = datetime.datetime.utcnow()
-                db.session.add(proxy)
-        db.session.commit()
 
-
+    def import_state(self, state: Any) -> None:
-def destroy_expired_proxies():
+        pass
-    cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
-    proxies = Proxy.query.filter(
-        Proxy.destroyed.is_(None),
-        Proxy.provider == "fastly",
-        Proxy.deprecated < cutoff
-    ).all()
-    for proxy in proxies:
-        proxy.destroyed = datetime.datetime.utcnow()
-        proxy.updated = datetime.datetime.utcnow()
-    db.session.commit()
-
-
-def generate_terraform():
-    filename = os.path.join(
-        app.config['TERRAFORM_DIRECTORY'],
-        'fastly',
-        'main.tf'
-    )
-    tmpl = jinja2.Template(TEMPLATE)
-    rendered = tmpl.render(
-        aws_access_key=app.config['AWS_ACCESS_KEY'],
-        aws_secret_key=app.config['AWS_SECRET_KEY'],
-        fastly_api_key=app.config['FASTLY_API_KEY'],
-        groups=Group.query.all()
-    )
-    with open(filename, 'w') as out:
-        out.write(rendered)
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        create_missing_proxies()
-        destroy_expired_proxies()
-        generate_terraform()