From 24e464653bd00288edda338eb7a2daaeb228ecb1 Mon Sep 17 00:00:00 2001 From: Iain Learmonth Date: Sun, 26 Feb 2023 15:06:40 +0000 Subject: [PATCH] feat(bridge): allow random provider selection --- app/portal/bridgeconf.py | 17 ++++++++++++- app/terraform/block/bridge.py | 2 +- app/terraform/bridge/meta.py | 46 +++++++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/app/portal/bridgeconf.py b/app/portal/bridgeconf.py index 42348fc..e236bd3 100644 --- a/app/portal/bridgeconf.py +++ b/app/portal/bridgeconf.py @@ -10,7 +10,7 @@ from wtforms.validators import DataRequired, NumberRange from app.extensions import db from app.models.base import Pool -from app.models.bridges import BridgeConf +from app.models.bridges import BridgeConf, ProviderAllocation from app.portal.util import response_404, view_lifecycle bp = Blueprint("bridgeconf", __name__) @@ -36,6 +36,12 @@ class NewBridgeConfForm(FlaskForm): # type: ignore expiry_hours = IntegerField('Expiry Timer (hours)', description=("The number of hours to wait after a bridge is deprecated before its " "destruction.")) + provider_allocation = SelectField('Provider Allocation Method', + description="How to allocate new bridges to providers.", + choices=[ + ("COST", "Use cheapest provider first"), + ("RANDOM", "Use providers randomly"), + ]) submit = SubmitField('Save Changes') @@ -51,6 +57,12 @@ class EditBridgeConfForm(FlaskForm): # type: ignore expiry_hours = IntegerField('Expiry Timer (hours)', description=("The number of hours to wait after a bridge is deprecated before its " "destruction.")) + provider_allocation = SelectField('Provider Allocation Method', + description="How to allocate new bridges to providers.", + choices=[ + ("COST", "Use cheapest provider first"), + ("RANDOM", "Use providers randomly"), + ]) submit = SubmitField('Save Changes') @@ -85,6 +97,7 @@ def bridgeconf_new(group_id: Optional[int] = None) -> ResponseReturnValue: bridgeconf.target_number = form.target_number.data bridgeconf.max_number = form.max_number.data bridgeconf.expiry_hours = form.expiry_hours.data + bridgeconf.provider_allocation = ProviderAllocation[form.provider_allocation.data] bridgeconf.created = datetime.utcnow() bridgeconf.updated = datetime.utcnow() try: @@ -115,12 +128,14 @@ def bridgeconf_edit(bridgeconf_id: int) -> ResponseReturnValue: target_number=bridgeconf.target_number, max_number=bridgeconf.max_number, expiry_hours=bridgeconf.expiry_hours, + provider_allocation=bridgeconf.provider_allocation.name, ) if form.validate_on_submit(): bridgeconf.description = form.description.data bridgeconf.target_number = form.target_number.data bridgeconf.max_number = form.max_number.data bridgeconf.expiry_hours = form.expiry_hours.data + bridgeconf.provider_allocation = ProviderAllocation[form.provider_allocation.data] bridgeconf.updated = datetime.utcnow() try: db.session.commit() diff --git a/app/terraform/block/bridge.py b/app/terraform/block/bridge.py index d4162fc..c082a64 100644 --- a/app/terraform/block/bridge.py +++ b/app/terraform/block/bridge.py @@ -25,7 +25,7 @@ class BlockBridgeAutomation(BaseAutomation): super().__init__(*args, **kwargs) def perform_deprecations(self, ids: List[str], bridge_select_func: Callable[[str], Optional[Bridge]] - ) -> List[Tuple[str, str]]: + ) -> List[Tuple[str, str, str]]: rotated = [] for id_ in ids: bridge = bridge_select_func(id_) diff --git a/app/terraform/bridge/meta.py b/app/terraform/bridge/meta.py index 094f5b8..d920dc1 100644 --- a/app/terraform/bridge/meta.py +++ b/app/terraform/bridge/meta.py @@ -1,9 +1,10 @@ import datetime import logging +import random from typing import Tuple, List from app import db -from app.models.bridges import BridgeConf, Bridge +from app.models.bridges import BridgeConf, Bridge, ProviderAllocation from app.models.cloud import CloudProvider, CloudAccount from app.terraform import BaseAutomation @@ -40,11 +41,11 @@ def create_bridges_in_account(bridgeconf: BridgeConf, account: CloudAccount, cou return created -def create_bridges(bridgeconf: BridgeConf, count: int) -> int: +def create_bridges_by_cost(bridgeconf: BridgeConf, count: int) -> int: """ - Creates a bridge resource for the given bridge configuration. + Creates bridge resources for the given bridge configuration using the cheapest available provider. """ - logging.debug("Creating %s bridges for configuration %s", count, bridgeconf.id) + logging.debug("Creating %s bridges by cost for configuration %s", count, bridgeconf.id) created = 0 for provider in BRIDGE_PROVIDERS: if created >= count: @@ -54,13 +55,48 @@ def create_bridges(bridgeconf: BridgeConf, count: int) -> int: CloudAccount.destroyed.is_(None), CloudAccount.enabled.is_(True), CloudAccount.provider == provider, - ): + ).all(): logging.info("Creating bridges in %s", account) created += create_bridges_in_account(bridgeconf, account, count - created) logging.debug("Created %s bridges", created) return created +def _accounts_with_room() -> List[CloudAccount]: + accounts = CloudAccount.query.filter( + CloudAccount.destroyed.is_(None), + CloudAccount.enabled.is_(True), + ).all() + accounts_with_room: List[CloudAccount] = [] + for account in accounts: + if len(active_bridges_in_account(account)) < account.max_instances: + accounts_with_room.append(account) + return accounts_with_room + + +def create_bridges_by_random(bridgeconf: BridgeConf, count: int) -> int: + """ + Creates bridge resources for the given bridge configuration using random providers. + """ + logging.debug("Creating %s bridges by random for configuration %s", count, bridgeconf.id) + created = 0 + while candidate_accounts := _accounts_with_room(): + # Not security-critical random number generation + account = random.choice(candidate_accounts) # nosec: B311 + create_bridges_in_account(bridgeconf, account, 1) + created += 1 + if created == count: + return count + return created # No account with room + + +def create_bridges(bridgeconf: BridgeConf, count: int) -> int: + if bridgeconf.provider_allocation == ProviderAllocation.COST: + return create_bridges_by_cost(bridgeconf, count) + else: + return create_bridges_by_random(bridgeconf, count) + + def deprecate_bridges(bridgeconf: BridgeConf, count: int, reason: str = "redundant") -> int: logging.debug("Deprecating %s bridges (%s) for configuration %s", count, reason, bridgeconf.id) deprecated = 0