From e4ca0f7ec9fc95b25eccac9cc507decc475c5254 Mon Sep 17 00:00:00 2001 From: Matt Hammerly Date: Tue, 17 Mar 2026 14:19:58 -0800 Subject: [PATCH] feat(objectstore): enable objectstore auth if key is configured --- src/launchpad/artifact_processor.py | 5 +- src/launchpad/service.py | 23 +++++++-- src/launchpad/utils/objectstore.py | 50 +++++++++++++++++++ tests/integration/test_kafka_service.py | 8 +-- .../unit/artifacts/test_artifact_processor.py | 12 ++--- 5 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 src/launchpad/utils/objectstore.py diff --git a/src/launchpad/artifact_processor.py b/src/launchpad/artifact_processor.py index 8185a8c2..e0e86296 100644 --- a/src/launchpad/artifact_processor.py +++ b/src/launchpad/artifact_processor.py @@ -45,6 +45,7 @@ from launchpad.tracing import request_context from launchpad.utils.file_utils import IdPrefix, id_from_bytes from launchpad.utils.logging import get_logger +from launchpad.utils.objectstore import create_objectstore_client from launchpad.utils.statsd import StatsdInterface, get_statsd logger = get_logger(__name__) @@ -88,9 +89,7 @@ def process_message( statsd = get_statsd() if artifact_processor is None: sentry_client = SentryClient(base_url=service_config.sentry_base_url) - objectstore_client = None - if service_config.objectstore_url is not None: - objectstore_client = ObjectstoreClient(service_config.objectstore_url) + objectstore_client = create_objectstore_client(service_config.objectstore_config) artifact_processor = ArtifactProcessor(sentry_client, statsd, objectstore_client) if service_config and project_id in service_config.projects_to_skip: diff --git a/src/launchpad/service.py b/src/launchpad/service.py index 84a2722b..6d3feb13 100644 --- a/src/launchpad/service.py +++ b/src/launchpad/service.py @@ -121,13 +121,23 @@ def _shutdown_server(self) -> None: self._server_thread.join(timeout=10) +@dataclass +class ObjectstoreConfig: + """Objectstore client configuration data.""" + + objectstore_url: str | None + key_id: str | None = None + key_file: str | None = None + token_expiry_seconds: int = 60 + + @dataclass class ServiceConfig: """Service configuration data.""" sentry_base_url: str projects_to_skip: list[str] - objectstore_url: str | None + objectstore_config: ObjectstoreConfig def get_service_config() -> ServiceConfig: @@ -135,7 +145,14 @@ def get_service_config() -> ServiceConfig: sentry_base_url = os.getenv("SENTRY_BASE_URL") projects_to_skip_str = os.getenv("PROJECT_IDS_TO_SKIP") projects_to_skip = projects_to_skip_str.split(",") if projects_to_skip_str else [] - objectstore_url = os.getenv("OBJECTSTORE_URL") + + objectstore_config = ObjectstoreConfig( + objectstore_url=os.getenv("OBJECTSTORE_URL"), + key_id=os.getenv("OBJECTSTORE_SIGNING_KEY_ID"), + key_file=os.getenv("OBJECTSTORE_SIGNING_KEY_FILE"), + ) + if expiry_seconds := os.getenv("OBJECTSTORE_TOKEN_EXPIRY_SECONDS"): + objectstore_config.token_expiry_seconds = int(expiry_seconds) if sentry_base_url is None: sentry_base_url = "http://getsentry.default" @@ -143,7 +160,7 @@ def get_service_config() -> ServiceConfig: return ServiceConfig( sentry_base_url=sentry_base_url, projects_to_skip=projects_to_skip, - objectstore_url=objectstore_url, + objectstore_config=objectstore_config, ) diff --git a/src/launchpad/utils/objectstore.py b/src/launchpad/utils/objectstore.py new file mode 100644 index 00000000..8e720a2f --- /dev/null +++ b/src/launchpad/utils/objectstore.py @@ -0,0 +1,50 @@ +from objectstore_client import ( + Client as ObjectstoreClient, +) +from objectstore_client import ( + Permission, + TokenGenerator, +) + +import launchpad + +from launchpad.utils.logging import get_logger + +logger = get_logger(__name__) + +_cached_keyfiles: dict[str, str] = {} + +TOKEN_PERMISSIONS: list[Permission] = [ + Permission.OBJECT_READ, + Permission.OBJECT_WRITE, + Permission.OBJECT_DELETE, +] + + +def _read_keyfile(path: str) -> str | None: + global _cached_keyfiles + if path not in _cached_keyfiles: + try: + with open(path) as f: + _cached_keyfiles[path] = f.read().strip() + except Exception: + logger.exception(f"Failed to load objectstore keyfile at {path}") + + return _cached_keyfiles.get(path) + + +def create_objectstore_client(config: "launchpad.service.ObjectstoreConfig") -> ObjectstoreClient | None: + if not config.objectstore_url: + return None + + token_generator = None + if config.key_id and config.key_file: + if secret_key := _read_keyfile(config.key_file): + token_generator = TokenGenerator( + config.key_id, + secret_key, + config.token_expiry_seconds, + TOKEN_PERMISSIONS, + ) + + return ObjectstoreClient(config.objectstore_url, token_generator=token_generator) diff --git a/tests/integration/test_kafka_service.py b/tests/integration/test_kafka_service.py index 2084c24e..c0a4df0f 100644 --- a/tests/integration/test_kafka_service.py +++ b/tests/integration/test_kafka_service.py @@ -12,7 +12,7 @@ from launchpad.artifact_processor import ArtifactProcessor from launchpad.constants import PREPROD_ARTIFACT_EVENTS_TOPIC from launchpad.kafka import LaunchpadKafkaConsumer, create_kafka_consumer, get_kafka_config -from launchpad.service import LaunchpadService, ServiceConfig, get_service_config +from launchpad.service import LaunchpadService, ObjectstoreConfig, ServiceConfig, get_service_config from launchpad.utils.statsd import FakeStatsd @@ -176,7 +176,7 @@ def test_process_message_with_skipped_project(self): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=["skip-project"], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) with patch.object(ArtifactProcessor, "process_artifact") as mock_process: @@ -196,7 +196,7 @@ def test_process_message_with_allowed_project(self): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=["other-project"], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) with patch.object(ArtifactProcessor, "process_artifact") as mock_process: @@ -225,7 +225,7 @@ def test_process_message_error_handling(self): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=[], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) with patch.object(ArtifactProcessor, "process_artifact", side_effect=RuntimeError("Test error")): diff --git a/tests/unit/artifacts/test_artifact_processor.py b/tests/unit/artifacts/test_artifact_processor.py index e5c75779..82f9361d 100644 --- a/tests/unit/artifacts/test_artifact_processor.py +++ b/tests/unit/artifacts/test_artifact_processor.py @@ -8,7 +8,7 @@ ProcessingErrorMessage, ) from launchpad.sentry_client import SentryClient, SentryClientError -from launchpad.service import ServiceConfig +from launchpad.service import ServiceConfig, ObjectstoreConfig from launchpad.utils.statsd import FakeStatsd @@ -146,7 +146,7 @@ def test_process_message_ios(self, mock_process, mock_sentry_client): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=[], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) ArtifactProcessor.process_message( @@ -182,7 +182,7 @@ def test_process_message_android(self, mock_process, mock_sentry_client): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=[], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) ArtifactProcessor.process_message( @@ -218,7 +218,7 @@ def test_process_message_error(self, mock_process, mock_sentry_client): service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=[], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) mock_process.side_effect = RuntimeError("Download failed: HTTP 404") @@ -252,7 +252,7 @@ def test_process_message_project_skipped(self, mock_process, mock_sentry_client) service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=["skip-project-1", "skip-project-2"], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) ArtifactProcessor.process_message( @@ -276,7 +276,7 @@ def test_process_message_project_not_skipped(self, mock_process, mock_sentry_cli service_config = ServiceConfig( sentry_base_url="http://test.sentry.io", projects_to_skip=["other-project"], - objectstore_url="http://test.objectstore.io", + objectstore_config=ObjectstoreConfig(objectstore_url="http://test.objectstore.io"), ) ArtifactProcessor.process_message(