proxy: adds support for fastly provider

app/cli/automate.py

@@ -28,6 +28,7 @@ from app.terraform.list.gitlab import ListGitlabAutomation
 from app.terraform.list.s3 import ListS3Automation
 from app.terraform.proxy.azure_cdn import ProxyAzureCdnAutomation
 from app.terraform.proxy.cloudfront import ProxyCloudfrontAutomation
+from app.terraform.proxy.fastly import ProxyFastlyAutomation
 
 jobs = {
     x.short_name: x
@@ -50,7 +51,8 @@ jobs = {
         ListGitlabAutomation,
         ListS3Automation,
         ProxyAzureCdnAutomation,
-        ProxyCloudfrontAutomation
+        ProxyCloudfrontAutomation,
+        ProxyFastlyAutomation
     ]
 }
 

app/terraform/proxy/fastly.py

@@ -1,144 +1,117 @@
-# type: ignore
+from typing import Any
-# TODO: This module doesn't work at all
 
-import datetime
+from app.terraform.proxy import ProxyAutomation
-import os
-import random
-import string
 
-import jinja2
-import tldextract
 
-from app import app
+class ProxyFastlyAutomation(ProxyAutomation):
-from app.extensions import db
+    short_name = "proxy_fastly"
-from app.models.base import Group
+    description = "Deploy proxies to Fastly"
-from app.models.mirrors import Origin, Proxy
+    provider = "fastly"
+    subgroup_max = 5
 
-TEMPLATE = """
+    template_parameters = [
-terraform {
+        "aws_access_key",
-  required_providers {
+        "aws_secret_key",
-    aws = {
+        "fastly_api_key"
-      version = "~> 4.4.0"
+    ]
+
+    template = """
+    terraform {
+      required_providers {
+        aws = {
+          version = "~> 4.4.0"
+        }
+        fastly = {
+          source  = "fastly/fastly"
+          version = ">= 1.1.1"
+        }
+      }
     }
-    fastly = {
+
-      source  = "fastly/fastly"
+    provider "aws" {
-      version = ">= 1.1.1"
+      access_key = "{{ aws_access_key }}"
+      secret_key = "{{ aws_secret_key }}"
+      region = "us-east-2"
     }
-  }
-}
 
-provider "aws" {
+    provider "fastly" {
-  access_key = "{{ aws_access_key }}"
+      api_key = "{{ fastly_api_key }}"
-  secret_key = "{{ aws_secret_key }}"
+    }
-  region = "us-east-1"
-}
 
-provider "fastly" {
+    {% for group in groups %}
-  api_key = "{{ fastly_api_key }}"
+    module "label_{{ group.id }}" {
-}
+      source  = "cloudposse/label/null"
+      version = "0.25.0"
+      namespace = "bc"
+      tenant = "{{ group.group_name }}"
+      label_order = ["namespace", "tenant", "name", "attributes"]
+    }
 
-{% for group in groups %}
+    module "log_bucket_{{ group.id }}" {
-module "label_{{ group.id }}" {
+      source = "cloudposse/s3-log-storage/aws"
-  source  = "cloudposse/label/null"
+      version = "0.28.0"
-  version = "0.25.0"
+      context = module.label_{{ group.id }}.context
-  namespace = "bc"
+      name = "logs"
-  tenant = "{{ group.group_name }}"
+      attributes = ["fastly"]
-  label_order = ["namespace", "tenant", "name", "attributes"]
+      acl                      = "private"
-}
+      standard_transition_days = 30
+      glacier_transition_days  = 60
+      expiration_days          = 90
+    }
 
-module "log_bucket_{{ group.id }}" {
+    {% if group.id == 3 %}
-  source = "cloudposse/s3-log-storage/aws"
+    {% for subgroup in subgroups[group.id] %}
-  version = "0.28.0"
+    resource "fastly_service_vcl" "service_{{ group.id }}_{{ subgroup }}" {
-  context = module.label_{{ group.id }}.context
+      name = "${module.label_{{ group.id }}.id}-{{ subgroup }}"
-  name = "logs"
-  attributes = ["fastly"]
-  acl                      = "private"
-  standard_transition_days = 30
-  glacier_transition_days  = 60
-  expiration_days          = 90
-}
 
-{% if group.id == 3 %}
+      {% for origin in group.origins %}
-resource "fastly_service_vcl" "service_{{ group.id }}" {
+      {% for proxy in origin.proxies %}
-  name = module.label_{{ group.id }}.id
+      {% if proxy.destroyed == None and proxy.provider == "fastly" and proxy.psg == subgroup %}
+      domain {
+        name    = "{{ proxy.slug }}.global.ssl.fastly.net"
+        comment = "Mirror"
+      }
 
-  {% for origin in group.origins %}
+      backend {
-  {% for proxy in origin.proxies %}
+        address       = "{{ proxy.origin.domain_name }}"
-  {% if proxy.destroyed == None and proxy.provider == "fastly" %}
+        name          = "{{ proxy.origin.domain_name.replace(".", "_") }}_{{ proxy.id }}"
-  domain {
+        port          = 443
-    name    = "{{ proxy.slug }}.global.ssl.fastly.com"
+        ssl_hostname  = "{{ proxy.origin.domain_name }}"
-    comment = "Mirror"
+        override_host = "{{ proxy.origin.domain_name }}"
-  }
+      }
-  {% endif %}
+      {% endif %}
-  {% endfor %}
+      {% endfor %}{# proxy #}
+      {% endfor %}{# origin #}
 
-  backend {
+      snippet {
-    address       = "{{ origin.domain_name }}"
+        name = "director"
-    name          = "{{ origin.description }}"
+        content = <<EOV
-    port          = 443
+            if (req.http.host == "") {
-    override_host = "{{ origin.domain_name }}"
+                error 400 "No host header";
-  }
+            }
-  {% endfor %}
+            {% for origin in group.origins %}
-}
+            {% for proxy in origin.proxies %}
-{% endif %}
+            {% if proxy.destroyed == None and proxy.provider == "fastly" and proxy.psg == subgroup %}
-{% endfor %}
+            else if (req.http.host == "{{ proxy.slug }}.global.ssl.fastly.net") {
-"""
+              set req.backend = F_{{ origin.domain_name.replace(".", "_") }}_{{ proxy.id }};
+              if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") {
+                return(pass);
+              }
+              return(lookup);
+            }
+            {% endif %}{% endfor %}{% endfor %}
+            else {
+                error 400 "Unknown host header";
+            }
+            EOV
+        type = "recv"
+      }
+    }
+    {% endfor %}{# subgroup #}
+    {% endif %}{# group.id == 3 #}
 
+    {% endfor %}{# group #}
+    """
 
-def create_missing_proxies():
+    def import_state(self, state: Any) -> None:
-    with app.app_context():
+        pass
-        origins = Origin.query.filter(Origin.group_id == 3).all()
-        for origin in origins:
-            azure_cdn_proxies = [
-                x for x in origin.proxies
-                if x.provider == "fastly" and x.deprecated is None and x.destroyed is None
-            ]
-            if not azure_cdn_proxies:
-                proxy = Proxy()
-                proxy.origin_id = origin.id
-                proxy.provider = "fastly"
-                # The random usage below is good enough for its purpose: to create a slug that
-                # hasn't been used before.
-                proxy.slug = tldextract.extract(origin.domain_name).domain[:5] + ''.join(
-                    random.choices(string.ascii_lowercase, 12))  # nosec
-                proxy.added = datetime.datetime.utcnow()
-                proxy.updated = datetime.datetime.utcnow()
-                db.session.add(proxy)
-        db.session.commit()
-
-
-def destroy_expired_proxies():
-    cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=3)
-    proxies = Proxy.query.filter(
-        Proxy.destroyed.is_(None),
-        Proxy.provider == "fastly",
-        Proxy.deprecated < cutoff
-    ).all()
-    for proxy in proxies:
-        proxy.destroyed = datetime.datetime.utcnow()
-        proxy.updated = datetime.datetime.utcnow()
-    db.session.commit()
-
-
-def generate_terraform():
-    filename = os.path.join(
-        app.config['TERRAFORM_DIRECTORY'],
-        'fastly',
-        'main.tf'
-    )
-    tmpl = jinja2.Template(TEMPLATE)
-    rendered = tmpl.render(
-        aws_access_key=app.config['AWS_ACCESS_KEY'],
-        aws_secret_key=app.config['AWS_SECRET_KEY'],
-        fastly_api_key=app.config['FASTLY_API_KEY'],
-        groups=Group.query.all()
-    )
-    with open(filename, 'w') as out:
-        out.write(rendered)
-
-
-if __name__ == "__main__":
-    with app.app_context():
-        create_missing_proxies()
-        destroy_expired_proxies()
-        generate_terraform()