jasima/majuna
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: abstracting cloud providers

Browse source
This commit is contained in:
Iain Learmonth 2023-02-26 12:52:08 +00:00
parent af36a545a1
commit 0a72aeed96
18 changed files with 629 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

37
app/terraform/bridge/__init__.py
View file

@ -1,14 +1,38 @@
import datetime
import os
import sys
from typing import Optional, Any, List
from typing import Optional, Any, List, Tuple
from sqlalchemy import select
from app import app
from app.extensions import db
from app.models.bridges import BridgeConf, Bridge
from app.models.base import Group
from app.models.bridges import Bridge, BridgeConf
from app.models.cloud import CloudAccount, CloudProvider
from app.terraform.terraform import TerraformAutomation
BridgeResourceRow = List[Tuple[Bridge, BridgeConf, CloudAccount]]
def active_bridges_by_provider(provider: CloudProvider) -> List[BridgeResourceRow]:
stmt = select(Bridge, BridgeConf, CloudAccount).join_from(Bridge, BridgeConf).join_from(Bridge, CloudAccount).where(
CloudAccount.provider == provider,
Bridge.destroyed.is_(None),
)
bridges: List[BridgeResourceRow] = db.session.execute(stmt).all()
return bridges
def recently_destroyed_bridges_by_provider(provider: CloudProvider) -> List[BridgeResourceRow]:
cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=72)
stmt = select(Bridge, BridgeConf, CloudAccount).join_from(Bridge, BridgeConf).join_from(Bridge, CloudAccount).where(
CloudAccount.provider == provider,
Bridge.destroyed.is_not(None),
Bridge.destroyed >= cutoff,
)
bridges: List[BridgeResourceRow] = db.session.execute(stmt).all()
return bridges
class BridgeAutomation(TerraformAutomation):
template: str
@ -33,11 +57,8 @@ class BridgeAutomation(TerraformAutomation):
def tf_generate(self) -> None:
self.tf_write(
self.template,
groups=Group.query.all(),
bridgeconfs=BridgeConf.query.filter(
BridgeConf.destroyed.is_(None),
BridgeConf.provider == self.provider
).all(),
active_resources=active_bridges_by_provider(self.provider),
destroyed_resources=recently_destroyed_bridges_by_provider(self.provider),
global_namespace=app.config['GLOBAL_NAMESPACE'],
terraform_modules_path=os.path.join(*list(os.path.split(app.root_path))[:-1], 'terraform-modules'),
backend_config=f"""backend "http" {{

65
app/terraform/bridge/aws.py
View file

@ -1,15 +1,15 @@
from app.models.cloud import CloudProvider
from app.terraform.bridge import BridgeAutomation
class BridgeAWSAutomation(BridgeAutomation):
short_name = "bridge_aws"
description = "Deploy Tor bridges on AWS Lightsail"
provider = "aws"
description = "Deploy Tor bridges on AWS EC2"
provider = CloudProvider.AWS
template_parameters = [
"aws_access_key",
"aws_secret_key",
"ssh_public_key_path"
"ssh_public_key_path",
"ssh_private_key_path",
]
template = """
@ -22,37 +22,42 @@ class BridgeAWSAutomation(BridgeAutomation):
}
}
provider "aws" {
access_key = "{{ aws_access_key }}"
secret_key = "{{ aws_secret_key }}"
region = "us-east-1"
}
locals {
ssh_key = file("{{ ssh_public_key_path }}")
ssh_public_key = "{{ ssh_public_key_path }}"
ssh_private_key = "{{ ssh_public_key_path }}"
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
version = "0.25.0"
namespace = "{{ global_namespace }}"
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
{% for resource in destroyed_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "aws" {
access_key = "{{ account.credentials['aws_access_key'] }}"
secret_key = "{{ account.credentials['aws_secret_key'] }}"
region = "{{ account.credentials['aws_region'] }}"
alias = "account_{{ bridge.id }}"
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
{% for resource in resources %}
{% set bridge, bridgeconf, account = resource %}
provider "aws" {
access_key = "{{ account.credentials['aws_access_key'] }}"
secret_key = "{{ account.credentials['aws_secret_key'] }}"
region = "{{ account.credentials['aws_region'] }}"
alias = "account_{{ bridge.id }}"
}
module "bridge_{{ bridge.id }}" {
source = "{{ terraform_modules_path }}/terraform-aws-tor-bridge"
ssh_key = local.ssh_key
contact_info = "hi"
context = module.label_{{ bridgeconf.group.id }}.context
name = "br"
attributes = ["{{ bridge.id }}"]
distribution_method = "{{ bridge.conf.method }}"
providers = {
aws = aws.account_{{ bridge.id }}
}
source = "{{ terraform_modules_path }}/terraform-aws-tor-bridge"
ssh_public_key = local.ssh_public_key
ssh_private_key = local.ssh_private_key
contact_info = "hi"
namespace = "{{ global_namespace }}"
name = "bridge"
attributes = ["{{ bridge.id }}"]
distribution_method = "{{ bridgeconf.method }}"
}
output "bridge_hashed_fingerprint_{{ bridge.id }}" {
@ -63,7 +68,5 @@ class BridgeAWSAutomation(BridgeAutomation):
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true
}
{% endif %}
{% endfor %}
{% endfor %}
"""

63
app/terraform/bridge/gandi.py
View file

@ -1,15 +1,13 @@
from app.models.cloud import CloudProvider
from app.terraform.bridge import BridgeAutomation
class BridgeGandiAutomation(BridgeAutomation):
short_name = "bridge_gandi"
description = "Deploy Tor bridges on GandiCloud VPS"
provider = "gandi"
provider = CloudProvider.GANDI
template_parameters = [
"gandi_openstack_user",
"gandi_openstack_password",
"gandi_openstack_tenant_name",
"ssh_public_key_path",
"ssh_private_key_path"
]
@ -25,43 +23,50 @@ class BridgeGandiAutomation(BridgeAutomation):
}
}
locals {
public_ssh_key = "{{ ssh_public_key_path }}"
private_ssh_key = "{{ ssh_private_key_path }}"
}
{% for resource in destroyed_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "openstack" {
auth_url = "https://keystone.sd6.api.gandi.net:5000/v3"
user_domain_name = "public"
project_domain_name = "public"
user_name = "{{ gandi_openstack_user }}"
password = "{{ gandi_openstack_password }}"
tenant_name = "{{ gandi_openstack_tenant_name }}"
user_name = "{{ account.credentials["gandi_openstack_user"] }}"
password = "{{ account.credentials["gandi_openstack_password"] }}"
tenant_name = "{{ account.credentials["gandi_openstack_tenant_id"] }}"
region = "FR-SD6"
}
locals {
public_ssh_key = file("{{ ssh_public_key_path }}")
private_ssh_key = file("{{ ssh_private_key_path }}")
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
version = "0.25.0"
namespace = "{{ global_namespace }}"
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
alias = "account_{{ bridge.id }}"
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
{% for resource in active_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "openstack" {
auth_url = "https://keystone.sd6.api.gandi.net:5000/v3"
user_domain_name = "public"
project_domain_name = "public"
user_name = "{{ account.credentials["gandi_openstack_user"] }}"
password = "{{ account.credentials["gandi_openstack_password"] }}"
tenant_name = "{{ account.credentials["gandi_openstack_tenant_id"] }}"
region = "FR-SD6"
alias = "account_{{ bridge.id }}"
}
module "bridge_{{ bridge.id }}" {
providers = {
openstack = openstack.account_{{ bridge.id }}
}
source = "{{ terraform_modules_path }}/terraform-openstack-tor-bridge"
context = module.label_{{ bridgeconf.group.id }}.context
name = "br"
namespace = "{{ global_namespace }}"
name = "bridge"
attributes = ["{{ bridge.id }}"]
ssh_key = local.public_ssh_key
ssh_private_key = local.private_ssh_key
contact_info = "hi"
distribution_method = "{{ bridge.conf.method }}"
contact_info = "did not write the code to populate yet"
distribution_method = "{{ bridgeconf.method }}"
image_name = "Debian 11 Bullseye"
flavor_name = "V-R1"
@ -77,7 +82,5 @@ class BridgeGandiAutomation(BridgeAutomation):
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true
}
{% endif %}
{% endfor %}
{% endfor %}
"""

67
app/terraform/bridge/hcloud.py
View file

@ -1,14 +1,15 @@
from app.models.cloud import CloudProvider
from app.terraform.bridge import BridgeAutomation
class BridgeHcloudAutomation(BridgeAutomation):
short_name = "bridge_hcloud"
description = "Deploy Tor bridges on Hetzner Cloud"
provider = "hcloud"
provider = CloudProvider.HCLOUD
template_parameters = [
"hcloud_token",
"ssh_private_key_path"
"ssh_private_key_path",
"ssh_public_key_path"
]
template = """
@ -26,36 +27,36 @@ class BridgeHcloudAutomation(BridgeAutomation):
}
}
provider "hcloud" {
token = "{{ hcloud_token }}"
}
locals {
ssh_private_key = file("{{ ssh_private_key_path }}")
ssh_private_key = "{{ ssh_private_key_path }}"
}
data "hcloud_datacenters" "ds" {
}
data "hcloud_server_type" "cx11" {
name = "cx11"
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
version = "0.25.0"
namespace = "{{ global_namespace }}"
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
{% for resource in destroyed_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "hcloud" {
token = "{{ account.credentials["hcloud_token"] }}"
alias = "account_{{ bridge.id }}"
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
{% for resource in active_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "hcloud" {
token = "{{ account.credentials["hcloud_token"] }}"
alias = "account_{{ bridge.id }}"
}
data "hcloud_datacenters" "ds_{{ bridge.id }}" {
provider = hcloud.account_{{ bridge.id }}
}
data "hcloud_server_type" "cx11_{{ bridge.id }}" {
provider = hcloud.account_{{ bridge.id }}
name = "cx11"
}
resource "random_shuffle" "datacenter_{{ bridge.id }}" {
input = [for s in data.hcloud_datacenters.ds.datacenters : s.name if contains(s.available_server_type_ids, data.hcloud_server_type.cx11.id)]
input = [for s in data.hcloud_datacenters.ds_{{ bridge.id }}.datacenters : s.name if contains(s.available_server_type_ids, data.hcloud_server_type.cx11_{{ bridge.id }}.id)]
result_count = 1
lifecycle {
@ -64,15 +65,17 @@ class BridgeHcloudAutomation(BridgeAutomation):
}
module "bridge_{{ bridge.id }}" {
providers = {
hcloud = hcloud.account_{{ bridge.id }}
}
source = "{{ terraform_modules_path }}/terraform-hcloud-tor-bridge"
datacenter = one(random_shuffle.datacenter_{{ bridge.id }}.result)
context = module.label_{{ bridgeconf.group.id }}.context
name = "br"
namespace = "{{ global_namespace }}"
name = "bridge"
attributes = ["{{ bridge.id }}"]
ssh_key_name = "bc"
ssh_private_key = local.ssh_private_key
contact_info = "hi"
distribution_method = "{{ bridge.conf.method }}"
contact_info = "this used to be sanitised and I did not write the code to populate it yet"
distribution_method = "{{ bridgeconf.method }}"
}
output "bridge_hashed_fingerprint_{{ bridge.id }}" {
@ -83,7 +86,5 @@ class BridgeHcloudAutomation(BridgeAutomation):
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true
}
{% endif %}
{% endfor %}
{% endfor %}
"""

69
app/terraform/bridge/meta.py
View file

@ -4,18 +4,40 @@ from typing import Tuple, List
from app import db
from app.models.bridges import BridgeConf, Bridge
from app.models.cloud import CloudProvider, CloudAccount
from app.terraform import BaseAutomation
from app.terraform.bridge.gandi import BridgeGandiAutomation
from app.terraform.bridge.hcloud import BridgeHcloudAutomation
from app.terraform.bridge.ovh import BridgeOvhAutomation
BRIDGE_PROVIDERS = {p.provider: p for p in [
BRIDGE_PROVIDERS = [
# In order of cost
BridgeHcloudAutomation,
BridgeGandiAutomation,
BridgeOvhAutomation,
# BridgeAWSAutomation, TODO: This module is broken right now
] if p.enabled}
CloudProvider.HCLOUD,
CloudProvider.GANDI,
CloudProvider.OVH,
CloudProvider.AWS,
]
def active_bridges_in_account(account: CloudAccount) -> List[Bridge]:
bridges: List[Bridge] = Bridge.query.filter(
Bridge.cloud_account_id == account.id,
Bridge.destroyed.is_(None),
).all()
return bridges
def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, count: int) -> int:
created = 0
while created < count and len(active_bridges_in_account(account)) < account.max_instances:
logging.debug("Creating bridge for configuration %s in account %s", bridgeconf.id, account)
bridge = Bridge()
bridge.pool_id = bridgeconf.pool.id
bridge.conf_id = bridgeconf.id
bridge.cloud_account = account
bridge.added = datetime.datetime.utcnow()
bridge.updated = datetime.datetime.utcnow()
logging.debug("Creating bridge %s", bridge)
db.session.add(bridge)
created += 1
return created
def create_bridges(bridgeconf: BridgeConf, count: int) -> int:
@ -24,24 +46,17 @@ def create_bridges(bridgeconf: BridgeConf, count: int) -> int:
"""
logging.debug("Creating %s bridges for configuration %s", count, bridgeconf.id)
created = 0
# TODO: deal with the fact that I created a dictionary and then forgot it wasn't ordered
while created < count:
for provider in BRIDGE_PROVIDERS:
if BRIDGE_PROVIDERS[provider].max_bridges > BRIDGE_PROVIDERS[provider].active_bridges_count():
logging.debug("Creating bridge for configuration %s with provider %s", bridgeconf.id, provider)
bridge = Bridge()
bridge.pool_id = bridgeconf.pool.id
bridge.conf_id = bridgeconf.id
bridge.provider = provider
bridge.added = datetime.datetime.utcnow()
bridge.updated = datetime.datetime.utcnow()
logging.debug("Creating bridge %s", bridge)
db.session.add(bridge)
created += 1
break
else:
logging.debug("No provider has available quota to create missing bridge for configuration %s",
bridgeconf.id)
for provider in BRIDGE_PROVIDERS:
if created >= count:
break
logging.info("Creating bridges in %s accounts", provider.description)
for account in CloudAccount.query.filter(
CloudAccount.destroyed.is_(None),
CloudAccount.enabled.is_(True),
CloudAccount.provider == provider,
):
logging.info("Creating bridges in %s", account)
created += create_bridges_in_account(bridgeconf, account, count - created)
logging.debug("Created %s bridges", created)
return created

80
app/terraform/bridge/ovh.py
View file

@ -1,18 +1,13 @@
from app.models.cloud import CloudProvider
from app.terraform.bridge import BridgeAutomation
class BridgeOvhAutomation(BridgeAutomation):
short_name = "bridge_ovh"
description = "Deploy Tor bridges on OVH Public Cloud"
provider = "ovh"
provider = CloudProvider.OVH
template_parameters = [
"ovh_cloud_application_key",
"ovh_cloud_application_secret",
"ovh_cloud_consumer_key",
"ovh_openstack_user",
"ovh_openstack_password",
"ovh_openstack_tenant_id",
"ssh_public_key_path",
"ssh_private_key_path"
]
@ -36,46 +31,50 @@ class BridgeOvhAutomation(BridgeAutomation):
}
}
locals {
public_ssh_key = "{{ ssh_public_key_path }}"
private_ssh_key = "{{ ssh_private_key_path }}"
}
{% for resource in destroyed_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "openstack" {
auth_url = "https://auth.cloud.ovh.net/v3/"
domain_name = "Default" # Domain name - Always at 'default' for OVHcloud
user_name = "{{ ovh_openstack_user }}"
password = "{{ ovh_openstack_password }}"
tenant_id = "{{ ovh_openstack_tenant_id }}"
user_name = "{{ account.credentials["ovh_openstack_user"] }}"
password = "{{ account.credentials["ovh_openstack_password"] }}"
tenant_id = "{{ account.credentials["ovh_openstack_tenant_id"] }}"
alias = "account_{{ bridge.id }}"
}
{% endfor %}
{% for resource in active_resources %}
{% set bridge, bridgeconf, account = resource %}
provider "openstack" {
auth_url = "https://auth.cloud.ovh.net/v3/"
domain_name = "Default" # Domain name - Always at 'default' for OVHcloud
user_name = "{{ account.credentials["ovh_openstack_user"] }}"
password = "{{ account.credentials["ovh_openstack_password"] }}"
tenant_id = "{{ account.credentials["ovh_openstack_tenant_id"] }}"
alias = "account_{{ bridge.id }}"
}
provider "ovh" {
endpoint = "ovh-eu"
application_key = "{{ ovh_cloud_application_key }}"
application_secret = "{{ ovh_cloud_application_secret }}"
consumer_key = "{{ ovh_cloud_consumer_key }}"
application_key = "{{ account.credentials["ovh_cloud_application_key"] }}"
application_secret = "{{ account.credentials["ovh_cloud_application_secret"] }}"
consumer_key = "{{ account.credentials["ovh_cloud_consumer_key"] }}"
alias = "account_{{ bridge.id }}"
}
locals {
public_ssh_key = file("{{ ssh_public_key_path }}")
private_ssh_key = file("{{ ssh_private_key_path }}")
}
data "ovh_cloud_project_regions" "regions" {
service_name = "{{ ovh_openstack_tenant_id }}"
data "ovh_cloud_project_regions" "regions_{{ bridge.id }}" {
provider = ovh.account_{{ bridge.id }}
service_name = "{{ account.credentials["ovh_openstack_tenant_id"] }}"
has_services_up = ["instance"]
}
{% for group in groups %}
module "label_{{ group.id }}" {
source = "cloudposse/label/null"
version = "0.25.0"
namespace = "{{ global_namespace }}"
tenant = "{{ group.group_name }}"
label_order = ["namespace", "tenant", "name", "attributes"]
}
{% endfor %}
{% for bridgeconf in bridgeconfs %}
{% for bridge in bridgeconf.bridges %}
{% if not bridge.destroyed %}
resource "random_shuffle" "region_{{ bridge.id }}" {
input = data.ovh_cloud_project_regions.regions.names
input = data.ovh_cloud_project_regions.regions_{{ bridge.id }}.names
result_count = 1
lifecycle {
@ -84,15 +83,18 @@ class BridgeOvhAutomation(BridgeAutomation):
}
module "bridge_{{ bridge.id }}" {
providers = {
openstack = openstack.account_{{ bridge.id }}
}
source = "{{ terraform_modules_path }}/terraform-openstack-tor-bridge"
region = one(random_shuffle.region_{{ bridge.id }}.result)
context = module.label_{{ bridgeconf.group.id }}.context
name = "br"
namespace = "{{ global_namespace }}"
name = "bridge"
attributes = ["{{ bridge.id }}"]
ssh_key = local.public_ssh_key
ssh_private_key = local.private_ssh_key
contact_info = "hi"
distribution_method = "{{ bridge.conf.method }}"
contact_info = "hello from onionland"
distribution_method = "{{ bridgeconf.method }}"
}
output "bridge_hashed_fingerprint_{{ bridge.id }}" {
@ -103,7 +105,5 @@ class BridgeOvhAutomation(BridgeAutomation):
value = module.bridge_{{ bridge.id }}.bridgeline
sensitive = true
}
{% endif %}
{% endfor %}
{% endfor %}
"""

3
app/terraform/list/__init__.py
View file

@ -43,6 +43,9 @@ class ListAutomation(TerraformAutomation):
in the templating of the Terraform configuration.
"""
provider: str # type: ignore[assignment]
# TODO: remove temporary override
def tf_generate(self) -> None:
if not self.working_dir:
raise RuntimeError("No working directory specified.")

3
app/terraform/proxy/__init__.py
View file

@ -71,6 +71,9 @@ class ProxyAutomation(TerraformAutomation):
The name of the cloud provider used by this proxy provider. Used to determine if this provider can be enabled.
"""
provider: str # type: ignore[assignment]
# TODO: Temporary override
@abstractmethod
def import_state(self, state: Any) -> None:
raise NotImplementedError()

3
app/terraform/terraform.py
View file

@ -3,6 +3,7 @@ import subprocess # nosec
from abc import abstractmethod
from typing import Any, Optional, Tuple
from app.models.cloud import CloudProvider
from app.terraform import BaseAutomation
@ -22,7 +23,7 @@ class TerraformAutomation(BaseAutomation):
Default parallelism for remote API calls.
"""
provider: str
provider: CloudProvider
"""
Short name for the provider used by this module.
"""