diff --git a/CHANGELOG.md b/CHANGELOG.md index d05847a..4d503d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.65.0] - 2026-10-09 + +### Added + +- `tilebox-workflows`: Add `client.storage_locations()` to find, list, create, and rename AWS S3, GCS, + Azure Blob, and local storage locations. +- `tilebox-workflows`: Format `str(storage_location)` as an S3 or GCS URL, Azure container resource ID, + or local path. + +### Changed + +- `tilebox-workflows`: Replace `client.automations().storage_locations()` with `client.storage_locations().all()`. +- `tilebox-workflows`: Rename `StorageType.S3` to `AWS_S3`, `FS` to `LOCAL`, and `AZURE` to `AZURE_BLOB`. + Rename `S3StorageLocation` to `AWSS3StorageLocation` + and `AzureStorageLocation` to `AzureBlobStorageLocation`. +- `tilebox-workflows`: Local storage reads raise `ValueError` if the root path is an empty string, missing, + or not a directory. Empty directories are valid; missing files raise `FileNotFoundError`. + ## [0.64.0] - 2026-10-06 ### Added @@ -601,7 +619,8 @@ the first client that does not cache data (since it's already on the local file - Released under the [MIT](https://opensource.org/license/mit) license. - Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc` -[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.64.0...HEAD +[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.65.0...HEAD +[0.65.0]: https://github.com/tilebox/tilebox-python/compare/v0.64.0...v0.65.0 [0.64.0]: https://github.com/tilebox/tilebox-python/compare/v0.63.2...v0.64.0 [0.63.2]: https://github.com/tilebox/tilebox-python/compare/v0.63.1...v0.63.2 [0.63.1]: https://github.com/tilebox/tilebox-python/compare/v0.63.0...v0.63.1 diff --git a/tilebox-workflows/tests/automations/test_storage_read.py b/tilebox-workflows/tests/automations/test_storage_read.py index 76fff6d..8c2c63d 100644 --- a/tilebox-workflows/tests/automations/test_storage_read.py +++ b/tilebox-workflows/tests/automations/test_storage_read.py @@ -23,9 +23,9 @@ from tilebox.workflows.cache import AmazonS3Cache from tilebox.workflows.client import Client from tilebox.workflows.data import ( - AzureStorageLocation, + AWSS3StorageLocation, + AzureBlobStorageLocation, RunnerContext, - S3StorageLocation, StorageLocation, StorageType, _default_azure_storage_client, @@ -110,8 +110,6 @@ def test_gcs_discovery_error_is_not_hidden_or_cached() -> None: def _azure_message() -> storage_pb.StorageLocation: return storage_pb.StorageLocation( id=uuid_to_uuid_message(_ID), - location="container-only", - type=storage_pb.STORAGE_TYPE_AZURE_BLOB, reference=storage_pb.StorageLocationReference( type=storage_pb.STORAGE_TYPE_AZURE_BLOB, azure_blob=storage_pb.AzureBlobReference(storage_account_resource_id=_ACCOUNT, container="sourcecontainer"), @@ -119,11 +117,11 @@ def _azure_message() -> storage_pb.StorageLocation: ) -@pytest.mark.parametrize("storage_type", [StorageType.GCS, StorageType.S3, StorageType.FS]) +@pytest.mark.parametrize("storage_type", [StorageType.GCS, StorageType.AWS_S3, StorageType.LOCAL]) def test_mixed_storage_location_ordering(storage_type: StorageType) -> None: # Azure and other storage locations sort together by ID before their bucket or container names. plain = StorageLocation(UUID(int=1), "z", storage_type) - azure = AzureStorageLocation(UUID(int=2), "a", StorageType.AZURE, storage_account_resource_id=_ACCOUNT) + azure = AzureBlobStorageLocation(UUID(int=2), "a", StorageType.AZURE_BLOB, storage_account_resource_id=_ACCOUNT) assert plain < azure assert azure > plain assert sorted([azure, plain]) == [plain, azure] @@ -132,8 +130,8 @@ def test_mixed_storage_location_ordering(storage_type: StorageType) -> None: def test_azure_trigger_read() -> None: # Deserialized Azure tasks read from the account and container supplied by the runner context. location = StorageLocation.from_message(_azure_message()) - assert isinstance(location, AzureStorageLocation) - assert location.type == StorageType.AZURE + assert isinstance(location, AzureBlobStorageLocation) + assert location.type == StorageType.AZURE_BLOB assert location.location == "sourcecontainer" assert location.storage_account_resource_id == _ACCOUNT assert StorageLocation.from_message(location.to_message()) == location @@ -145,7 +143,7 @@ def test_azure_trigger_read() -> None: context = RunnerContext(storage_locations=[location]) task = StorageEventTask().once(location, key) restored = StorageEventTask._deserialize(task._serialize(), context) - assert isinstance(restored.trigger.storage, AzureStorageLocation) + assert isinstance(restored.trigger.storage, AzureBlobStorageLocation) assert restored.trigger.storage.storage_account_resource_id == _ACCOUNT assert restored.trigger.storage.runner_context is context with patch.object(context, "azure_client", return_value=store) as client: @@ -211,24 +209,23 @@ def test_azure_explicit_credentials(variable: str, value: str, monkeypatch: pyte def test_azure_read_requires_account_metadata() -> None: # Missing or malformed Azure account resource IDs raise a clear error before a cloud request. - location = StorageLocation(_ID, "container-only", StorageType.AZURE) + location = StorageLocation(_ID, "container-only", StorageType.AZURE_BLOB) with pytest.raises(ValueError, match="missing its storage account resource ID"): location.read("file.txt") with pytest.raises(ValueError, match="Invalid Azure storage account resource ID"): RunnerContext().azure_client("https://sourceaccount.blob.core.windows.net", "container") -@pytest.mark.parametrize("metadata", ["bare", "reference"]) -def test_gcs_trigger_read(metadata: str) -> None: - # GCS tasks read bare bucket names using scoped google-auth credentials from either API metadata form. +def test_gcs_trigger_read() -> None: + # GCS tasks read the referenced bucket using scoped google-auth credentials. bucket = "storage-automation-test-gcp-2aa460e" message = storage_pb.StorageLocation( - id=uuid_to_uuid_message(_ID), location=bucket, type=storage_pb.STORAGE_TYPE_GCS_BUCKET + id=uuid_to_uuid_message(_ID), + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_GCS, + gcs_bucket=storage_pb.GCSBucketReference(project_id="test-project", bucket=bucket), + ), ) - if metadata == "reference": - message.reference.type = storage_pb.STORAGE_TYPE_GCS_BUCKET - message.reference.gcs_bucket.project_id = "test-project" - message.reference.gcs_bucket.bucket = bucket location = StorageLocation.from_message(message) context = RunnerContext(storage_locations=[location]) key = "nested/hello + %20 ü.txt" @@ -281,7 +278,7 @@ def test_s3_read_preserves_aws_profile(monkeypatch: pytest.MonkeyPatch, tmp_path expected = b"S3 blob contents\n" obstore.put(store, key, expected) with patch("obstore.store.S3Store", return_value=store) as constructor: - assert S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read(key) == expected + assert AWSS3StorageLocation(_ID, "aws-bucket", StorageType.AWS_S3, region="eu-west-2").read(key) == expected assert constructor.call_args.args == ("aws-bucket",) provider = constructor.call_args.kwargs["credential_provider"] assert provider()["access_key_id"] == "profile-key" @@ -301,7 +298,7 @@ def test_s3_read_preserves_default_session(monkeypatch: pytest.MonkeyPatch) -> N obstore.put(store, "file.txt", b"configured session") with patch("obstore.store.S3Store", return_value=store) as constructor: assert ( - S3StorageLocation(_ID, "aws-bucket", StorageType.S3, region="eu-west-2").read("file.txt") + AWSS3StorageLocation(_ID, "aws-bucket", StorageType.AWS_S3, region="eu-west-2").read("file.txt") == b"configured session" ) provider = constructor.call_args.kwargs["credential_provider"] @@ -316,12 +313,12 @@ def test_s3_reference_region_and_store_cache(monkeypatch: pytest.MonkeyPatch) -> message = storage_pb.StorageLocation( id=uuid_to_uuid_message(_ID), reference=storage_pb.StorageLocationReference( - type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + type=storage_pb.STORAGE_TYPE_AWS_S3, aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="source-bucket", region="eu-west-2"), ), ) location = StorageLocation.from_message(message) - assert isinstance(location, S3StorageLocation) + assert isinstance(location, AWSS3StorageLocation) assert StorageLocation.from_message(location.to_message()) == location context = RunnerContext(storage_locations=[location]) task = StorageEventTask().once(location, "file.txt") @@ -447,13 +444,49 @@ def log_message(self, format: str, *args: object) -> None: # noqa: A002 assert all("/eu-west-2/s3/" in authorization for authorization in authorizations) -def test_filesystem_read(tmp_path: Path) -> None: - # Filesystem reads return the contents of the file under the configured root directory. - key = "file.txt" +def test_local_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + working_directory = tmp_path / "work" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + key = "nested/hello + %20 ü.txt" expected = b"existing provider content" - storage = StorageLocation(_ID, str(tmp_path), StorageType.FS) + storage = StorageLocation.from_message( + storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), + reference=storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_LOCAL, + local=storage_pb.LocalReference(path=str(tmp_path)), + ), + ) + ) + (tmp_path / "nested").mkdir() (tmp_path / key).write_bytes(expected) - assert storage.read(key) == expected + context = RunnerContext(storage_locations=[storage]) + task = StorageEventTask().once(storage, key) + restored = StorageEventTask._deserialize(task._serialize(), context) + assert restored.trigger.storage.read(restored.trigger.location) == expected + with pytest.raises(FileNotFoundError): + storage.read("missing.txt") + + +@pytest.mark.parametrize("root_kind", ["empty_path", "missing", "file"]) +def test_local_read_requires_available_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, root_kind: str) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "file.txt").write_bytes(b"must not fall back to the current directory") + root = tmp_path / "root" + if root_kind == "file": + root.write_bytes(b"not a directory") + storage = StorageLocation(_ID, "" if root_kind == "empty_path" else str(root), StorageType.LOCAL) + with pytest.raises(ValueError, match="Local storage location root is not available"): + storage.read("file.txt") + if root_kind == "missing": + assert not root.exists() + + +def test_local_read_accepts_empty_directory(tmp_path: Path) -> None: + storage = StorageLocation(_ID, str(tmp_path), StorageType.LOCAL) + with pytest.raises(FileNotFoundError): + storage.read("missing.txt") @pytest.mark.parametrize( @@ -461,7 +494,7 @@ def test_filesystem_read(tmp_path: Path) -> None: [ ( storage_pb.StorageLocationReference( - type=storage_pb.STORAGE_TYPE_GCS_BUCKET, + type=storage_pb.STORAGE_TYPE_GCS, gcs_bucket=storage_pb.GCSBucketReference(project_id="project", bucket="bucket"), ), StorageType.GCS, @@ -469,17 +502,17 @@ def test_filesystem_read(tmp_path: Path) -> None: ), ( storage_pb.StorageLocationReference( - type=storage_pb.STORAGE_TYPE_AWS_S3_BUCKET, + type=storage_pb.STORAGE_TYPE_AWS_S3, aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket="bucket"), ), - StorageType.S3, + StorageType.AWS_S3, "bucket", ), ( storage_pb.StorageLocationReference( - type=storage_pb.STORAGE_TYPE_FILESYSTEM, filesystem=storage_pb.FilesystemReference(path="/data") + type=storage_pb.STORAGE_TYPE_LOCAL, local=storage_pb.LocalReference(path="/data") ), - StorageType.FS, + StorageType.LOCAL, "/data", ), ], @@ -503,8 +536,8 @@ def test_grpc_storage_location_service_route() -> None: patch("tilebox.workflows.client.open_channel", return_value=channel), patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), ): - locations = Client(token="test-key").automations().storage_locations() # noqa: S106 - assert isinstance(locations[0], AzureStorageLocation) + locations = Client(token="test-key").storage_locations().all() # noqa: S106 + assert isinstance(locations[0], AzureBlobStorageLocation) assert locations[0].storage_account_resource_id == _ACCOUNT assert "/workflows.v1.StorageLocationService/ListStorageLocations" in [ call.args[0] for call in channel.unary_unary.call_args_list @@ -520,7 +553,7 @@ def test_http1_storage_location_service_route() -> None: ) as list_locations, patch("tilebox.workflows.client.WorkflowTracer", return_value=NoopWorkflowTracer()), ): - locations = Client(token="test-key", transport="http1").automations().storage_locations() # noqa: S106 - assert isinstance(locations[0], AzureStorageLocation) + locations = Client(token="test-key", transport="http1").storage_locations().all() # noqa: S106 + assert isinstance(locations[0], AzureBlobStorageLocation) assert locations[0].storage_account_resource_id == _ACCOUNT list_locations.assert_called_once() diff --git a/tilebox-workflows/tests/runner/test_worker_concurrency.py b/tilebox-workflows/tests/runner/test_worker_concurrency.py index 107af5e..551425b 100644 --- a/tilebox-workflows/tests/runner/test_worker_concurrency.py +++ b/tilebox-workflows/tests/runner/test_worker_concurrency.py @@ -135,7 +135,7 @@ def storage_locations() -> list[MagicMock]: return [storage_location] client = MagicMock() - client.automations.return_value.storage_locations.side_effect = storage_locations + client.storage_locations.return_value.all.side_effect = storage_locations locations = LazyStorageLocations(client, RunnerContext()) first = threading.Thread(target=len, args=(locations,)) @@ -157,7 +157,7 @@ def read_locations() -> None: assert not first.is_alive() assert not second.is_alive() - client.automations.return_value.storage_locations.assert_called_once_with() + client.storage_locations.return_value.all.assert_called_once_with() assert list(locations) == [storage_location.id] diff --git a/tilebox-workflows/tests/storage_locations/test_client.py b/tilebox-workflows/tests/storage_locations/test_client.py new file mode 100644 index 0000000..8284f36 --- /dev/null +++ b/tilebox-workflows/tests/storage_locations/test_client.py @@ -0,0 +1,201 @@ +from dataclasses import replace +from datetime import UTC, datetime +from uuid import UUID + +import pytest +from google.protobuf.empty_pb2 import Empty + +from _tilebox.grpc.error import NotFoundError +from tilebox.datasets.query.pagination import Pagination +from tilebox.datasets.query.time_interval import datetime_to_timestamp +from tilebox.datasets.tilebox.v1.id_pb2 import ID +from tilebox.datasets.tilebox.v1.query_pb2 import Pagination as PaginationMessage +from tilebox.datasets.uuid import uuid_message_to_uuid, uuid_to_uuid_message +from tilebox.workflows.data import ( + AWSS3BucketReference, + AWSS3StorageLocation, + AzureBlobReference, + AzureBlobStorageLocation, + GCSBucketReference, + GCSStorageLocation, + LocalReference, + StorageEventType, + StorageLocation, + StorageLocationReference, + StorageType, +) +from tilebox.workflows.storage_locations import StorageLocationClient +from tilebox.workflows.storage_locations.data import ( + StorageSubscriptionEvent, + TriggeredJob, +) +from tilebox.workflows.storage_locations.service import StorageLocationService +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb + +_ID = UUID(int=1) +_OTHER_ID = UUID(int=2) +_ACCOUNT = "/subscriptions/example/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/account" + + +class MockStorageLocationService: + def __init__(self) -> None: + self.locations: dict[UUID, storage_pb.StorageLocation] = {} + self.events: dict[UUID, list[storage_pb.StorageSubscriptionEvent]] = {} + + def CreateStorageLocation(self, request: storage_pb.CreateStorageLocationRequest) -> storage_pb.StorageLocation: # noqa: N802 + location_id = UUID(int=len(self.locations) + 1) + location = storage_pb.StorageLocation( + id=uuid_to_uuid_message(location_id), name=request.name, reference=request.reference + ) + self.locations[location_id] = location + return location + + def GetStorageLocation(self, request: ID) -> storage_pb.StorageLocation: # noqa: N802 + try: + return self.locations[uuid_message_to_uuid(request)] + except KeyError: + raise NotFoundError("Unknown storage location") from None + + def ListStorageLocations(self, request: Empty) -> storage_pb.StorageLocations: # noqa: N802, ARG002 + return storage_pb.StorageLocations(locations=self.locations.values()) + + def UpdateStorageLocation(self, request: storage_pb.UpdateStorageLocationRequest) -> storage_pb.StorageLocation: # noqa: N802 + location = self.GetStorageLocation(request.storage_location_id) + location.name = request.name + return location + + def ListStorageSubscriptionEvents( # noqa: N802 + self, request: storage_pb.ListStorageSubscriptionEventsRequest + ) -> storage_pb.StorageSubscriptionEvents: + events = self.events.get(uuid_message_to_uuid(request.storage_location_id), []) + if request.page.HasField("starting_after"): + index = next(i for i, event in enumerate(events) if event.id == request.page.starting_after) + 1 + events = events[index:] + limit = request.page.limit or len(events) + return storage_pb.StorageSubscriptionEvents( + events=events[:limit], + next_page=PaginationMessage(limit=limit, starting_after=events[limit - 1].id) + if len(events) > limit + else None, + ) + + +@pytest.mark.parametrize( + ("reference", "expected", "expected_string"), + [ + ( + AWSS3BucketReference("s3-bucket", region="eu-west-2"), + AWSS3StorageLocation(_ID, "s3-bucket", StorageType.AWS_S3, name="Source", region="eu-west-2"), + "s3://s3-bucket", + ), + ( + GCSBucketReference("gcs-bucket", project_id="project", location="EU"), + GCSStorageLocation( + _ID, "gcs-bucket", StorageType.GCS, name="Source", project_id="project", bucket_location="EU" + ), + "gs://gcs-bucket", + ), + ( + AzureBlobReference(_ACCOUNT, "container", region="westeurope"), + AzureBlobStorageLocation( + _ID, + "container", + StorageType.AZURE_BLOB, + name="Source", + storage_account_resource_id=_ACCOUNT, + region="westeurope", + ), + f"{_ACCOUNT}/containers/container", + ), + ( + LocalReference("/data/source"), + StorageLocation(_ID, "/data/source", StorageType.LOCAL, name="Source"), + "/data/source", + ), + ], +) +def test_storage_location_operations( + reference: StorageLocationReference, expected: StorageLocation, expected_string: str +) -> None: + client = StorageLocationClient(StorageLocationService(MockStorageLocationService())) + assert client.all() == [] + created = client.create("Source", reference) + other = client.create("Other", LocalReference("/data/other")) + assert created == expected + assert str(created) == expected_string + message = created.to_message() + assert message.location == "" + assert message.type == storage_pb.STORAGE_TYPE_UNSPECIFIED + assert StorageLocation.from_message(message) == expected + message.location = "ignored-legacy-location" + message.type = storage_pb.STORAGE_TYPE_LOCAL if expected.type != StorageType.LOCAL else storage_pb.STORAGE_TYPE_GCS + assert StorageLocation.from_message(message) == expected + assert client.all() == [expected, other] + for identifier in (created, created.id, str(created.id)): + assert client.find(identifier) == expected + assert client.update(identifier, "Renamed") == replace(expected, name="Renamed") + assert client.find(other).name == "Other" + assert client.update(identifier, "Source") == expected + with pytest.raises(NotFoundError, match="Unknown storage location"): + client.find(UUID(int=99)) + + +def test_storage_location_requires_reference() -> None: + message = storage_pb.StorageLocation( + id=uuid_to_uuid_message(_ID), location="legacy-bucket", type=storage_pb.STORAGE_TYPE_AWS_S3 + ) + with pytest.raises(ValueError, match="Unsupported storage reference type"): + StorageLocation.from_message(message) + + +def test_storage_event_service_returns_one_page_and_cursor() -> None: + service = MockStorageLocationService() + storage_service = StorageLocationService(service) + event_time = datetime(2026, 10, 7, 12, 34, 56, 123456, tzinfo=UTC) + received_at = datetime(2026, 10, 8, 1, 2, 3, 654321, tzinfo=UTC) + service.events[_ID] = [ + storage_pb.StorageSubscriptionEvent( + id=uuid_to_uuid_message(UUID(int=10)), + object_key="nested/first.txt", + type=storage_pb.STORAGE_EVENT_TYPE_CREATED, + event_time=datetime_to_timestamp(event_time), + received_at=datetime_to_timestamp(received_at), + storage_subscription_id=uuid_to_uuid_message(UUID(int=20)), + triggered_jobs=[ + storage_pb.TriggeredJob( + job_id=uuid_to_uuid_message(UUID(int=30)), + automation_id=uuid_to_uuid_message(UUID(int=40)), + glob_pattern="nested/*.txt", + ) + ], + ), + storage_pb.StorageSubscriptionEvent( + id=uuid_to_uuid_message(UUID(int=11)), + object_key="second.bin", + type=storage_pb.STORAGE_EVENT_TYPE_CREATED, + event_time=datetime_to_timestamp(received_at), + received_at=datetime_to_timestamp(received_at), + storage_subscription_id=uuid_to_uuid_message(UUID(int=21)), + ), + ] + expected = [ + StorageSubscriptionEvent( + UUID(int=10), + "nested/first.txt", + StorageEventType.CREATED, + event_time, + received_at, + [TriggeredJob(UUID(int=30), UUID(int=40), "nested/*.txt")], + UUID(int=20), + ), + StorageSubscriptionEvent( + UUID(int=11), "second.bin", StorageEventType.CREATED, received_at, received_at, [], UUID(int=21) + ), + ] + first_page = storage_service.list_events(_ID, Pagination(limit=1)) + assert first_page.events == expected[:1] + assert first_page.next_page == Pagination(limit=1, starting_after=UUID(int=10)) + second_page = storage_service.list_events(_ID, first_page.next_page) + assert second_page.events == expected[1:] + assert second_page.next_page == Pagination() + assert storage_service.list_events(_OTHER_ID, Pagination()).events == [] diff --git a/tilebox-workflows/tests/tasks_data.py b/tilebox-workflows/tests/tasks_data.py index 75f1092..ba71ccf 100644 --- a/tilebox-workflows/tests/tasks_data.py +++ b/tilebox-workflows/tests/tasks_data.py @@ -30,12 +30,14 @@ from tilebox.workflows.data import ( Artifact, AutomationPrototype, - AzureStorageLocation, + AWSS3StorageLocation, + AzureBlobStorageLocation, Cluster, ComputedTask, CronTrigger, ExecutionStats, FilesystemNode, + GCSStorageLocation, Idling, Job, JobState, @@ -332,14 +334,18 @@ def storage_locations(draw: DrawFn) -> StorageLocation: location = draw(alphanumerical_text(min_size=4, max_size=30)) storage_type = draw(sampled_from(StorageType)) - if storage_type == StorageType.AZURE: + if storage_type == StorageType.AZURE_BLOB: account = draw(alphanumerical_text(min_size=3, max_size=24)) account_resource_id = ( f"/subscriptions/example/resourceGroups/storage/providers/Microsoft.Storage/storageAccounts/{account}" ) - return AzureStorageLocation( + return AzureBlobStorageLocation( storage_location_id, location, storage_type, storage_account_resource_id=account_resource_id ) + if storage_type == StorageType.AWS_S3: + return AWSS3StorageLocation(storage_location_id, location, storage_type) + if storage_type == StorageType.GCS: + return GCSStorageLocation(storage_location_id, location, storage_type) return StorageLocation(storage_location_id, location, storage_type) diff --git a/tilebox-workflows/tilebox/workflows/automations/client.py b/tilebox-workflows/tilebox/workflows/automations/client.py index f98e8f1..42dec93 100644 --- a/tilebox-workflows/tilebox/workflows/automations/client.py +++ b/tilebox-workflows/tilebox/workflows/automations/client.py @@ -18,14 +18,6 @@ class AutomationClient: def __init__(self, service: AutomationService) -> None: self._service = service - def storage_locations(self) -> list[StorageLocation]: - """List all available storage locations that can potentially be used as storage event triggers. - - Returns: - A list of all available storage locations. - """ - return self._service.list_storage_locations() - def all(self) -> list[AutomationPrototype]: """List all registered automations. diff --git a/tilebox-workflows/tilebox/workflows/automations/service.py b/tilebox-workflows/tilebox/workflows/automations/service.py index 06cdc50..4b6f163 100644 --- a/tilebox-workflows/tilebox/workflows/automations/service.py +++ b/tilebox-workflows/tilebox/workflows/automations/service.py @@ -9,39 +9,24 @@ from tilebox.datasets.uuid import uuid_to_uuid_message from tilebox.workflows.data import ( AutomationPrototype, - StorageLocation, ) from tilebox.workflows.workflows.v1.automation_pb2 import AutomationPrototype as AutomationPrototypeMessage from tilebox.workflows.workflows.v1.automation_pb2 import Automations, DeleteAutomationRequest from tilebox.workflows.workflows.v1.automation_pb2_grpc import AutomationServiceStub -from tilebox.workflows.workflows.v1.storage_location_pb2 import StorageLocations -from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub class AutomationService: - def __init__(self, channel: Channel | Any, storage_channel: Channel | Any = None) -> None: + def __init__(self, channel: Channel | Any) -> None: """ A wrapper around the AutomationServiceStub that provides a more pythonic interface and converts the protobuf messages to and from the data classes used in the rest of the tilebox-workflows codebase. Args: channel: The gRPC channel to use for the service. - storage_channel: Optional separate storage-location client for the Connect transport. """ self.service = ( with_pythonic_errors(AutomationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel ) - if storage_channel is None: - storage_channel = channel - self.storage_service = ( - with_pythonic_errors(StorageLocationServiceStub(storage_channel)) - if hasattr(storage_channel, "unary_unary") - else storage_channel - ) - - def list_storage_locations(self) -> builtins.list[StorageLocation]: - response: StorageLocations = self.storage_service.ListStorageLocations(Empty()) - return [StorageLocation.from_message(sl) for sl in response.locations] def list(self) -> builtins.list[AutomationPrototype]: response: Automations = self.service.ListAutomations(Empty()) diff --git a/tilebox-workflows/tilebox/workflows/automations/storage_event.py b/tilebox-workflows/tilebox/workflows/automations/storage_event.py index 9e27f81..355c8e3 100644 --- a/tilebox-workflows/tilebox/workflows/automations/storage_event.py +++ b/tilebox-workflows/tilebox/workflows/automations/storage_event.py @@ -13,7 +13,7 @@ from tilebox.workflows.workflows.v1.storage_location_pb2 import TriggeredStorageEvent as TriggeredStorageEventMessage _NOT_TRIGGERED = TriggeredStorageEvent( - StorageLocation(UUID(int=0), "", StorageType.FS), + StorageLocation(UUID(int=0), "", StorageType.LOCAL), StorageEventType.CREATED, "", ) diff --git a/tilebox-workflows/tilebox/workflows/client.py b/tilebox-workflows/tilebox/workflows/client.py index c19a167..d24b80c 100644 --- a/tilebox-workflows/tilebox/workflows/client.py +++ b/tilebox-workflows/tilebox/workflows/client.py @@ -23,6 +23,8 @@ from tilebox.workflows.runner.executor import LazyStorageLocations from tilebox.workflows.runner.task_runner import TaskRunner, _LeaseRenewer from tilebox.workflows.runner.task_service import TaskService +from tilebox.workflows.storage_locations.client import StorageLocationClient +from tilebox.workflows.storage_locations.service import StorageLocationService from tilebox.workflows.task import Task from tilebox.workflows.workflows.client import WorkflowClient from tilebox.workflows.workflows.service import WorkflowService @@ -207,7 +209,11 @@ def automations(self) -> AutomationClient: Returns: A client for the automations service. """ - return AutomationClient(AutomationService(self._automation_service, self._storage_location_service)) + return AutomationClient(AutomationService(self._automation_service)) + + def storage_locations(self) -> StorageLocationClient: + """Get a client for storage locations.""" + return StorageLocationClient(StorageLocationService(self._storage_location_service)) def _token_from_env(url: str, token: str | None) -> str | None: diff --git a/tilebox-workflows/tilebox/workflows/data.py b/tilebox-workflows/tilebox/workflows/data.py index f72dbb9..c19dccf 100644 --- a/tilebox-workflows/tilebox/workflows/data.py +++ b/tilebox-workflows/tilebox/workflows/data.py @@ -969,13 +969,67 @@ def from_message(cls, page: Any) -> "QueryJobSpansResponse": class StorageType(Enum): - GCS = storage_pb.STORAGE_TYPE_GCS_BUCKET # Google Cloud Storage - S3 = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET # Amazon Web Services S3 - FS = storage_pb.STORAGE_TYPE_FILESYSTEM # Local Filesystem - AZURE = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage + GCS = storage_pb.STORAGE_TYPE_GCS # Google Cloud Storage + AWS_S3 = storage_pb.STORAGE_TYPE_AWS_S3 # Amazon Web Services S3 + LOCAL = storage_pb.STORAGE_TYPE_LOCAL # Local Filesystem + AZURE_BLOB = storage_pb.STORAGE_TYPE_AZURE_BLOB # Azure Blob Storage -_STORAGE_TYPE_TO_ENUM = {storage_type.value: storage_type for storage_type in StorageType} +@dataclass(frozen=True) +class AWSS3BucketReference: + bucket: str + region: str = "" + + def to_message(self) -> storage_pb.StorageLocationReference: + return storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AWS_S3, + aws_s3_bucket=storage_pb.AWSS3BucketReference(bucket=self.bucket, region=self.region), + ) + + +@dataclass(frozen=True) +class GCSBucketReference: + bucket: str + project_id: str = "" + location: str = "" + + def to_message(self) -> storage_pb.StorageLocationReference: + return storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_GCS, + gcs_bucket=storage_pb.GCSBucketReference( + bucket=self.bucket, project_id=self.project_id, location=self.location + ), + ) + + +@dataclass(frozen=True) +class AzureBlobReference: + storage_account_resource_id: str + container: str + region: str = "" + + def to_message(self) -> storage_pb.StorageLocationReference: + return storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_AZURE_BLOB, + azure_blob=storage_pb.AzureBlobReference( + storage_account_resource_id=self.storage_account_resource_id, + container=self.container, + region=self.region, + ), + ) + + +@dataclass(frozen=True) +class LocalReference: + path: str + + def to_message(self) -> storage_pb.StorageLocationReference: + return storage_pb.StorageLocationReference( + type=storage_pb.STORAGE_TYPE_LOCAL, local=storage_pb.LocalReference(path=self.path) + ) + + +StorageLocationReference: TypeAlias = AWSS3BucketReference | GCSBucketReference | AzureBlobReference | LocalReference @total_ordering @@ -985,53 +1039,81 @@ class StorageLocation: location: str type: StorageType runner_context: "RunnerContext | None" = None + name: str = field(default="", kw_only=True) def __lt__(self, other: object) -> bool: if not isinstance(other, StorageLocation): return NotImplemented return (self.id, self.location, self.type.value) < (other.id, other.location, other.type.value) + def __str__(self) -> str: + match self.type: + case StorageType.AWS_S3: + return f"s3://{self.location}" + case StorageType.GCS: + return f"gs://{self.location}" + case StorageType.LOCAL: + return self.location + case _: + return self.type.name.lower().replace("_", "-") + @classmethod def from_message(cls, storage_location: storage_pb.StorageLocation) -> "StorageLocation": """Convert a StorageLocation protobuf message to a StorageLocation object.""" - location = storage_location.location - storage_type = storage_location.type - if storage_location.HasField("reference"): - reference = storage_location.reference - storage_type = reference.type - match storage_type: - case storage_pb.STORAGE_TYPE_AZURE_BLOB: - return AzureStorageLocation( - id=uuid_message_to_uuid(storage_location.id), - location=reference.azure_blob.container, - type=StorageType.AZURE, - storage_account_resource_id=reference.azure_blob.storage_account_resource_id, - ) - case storage_pb.STORAGE_TYPE_GCS_BUCKET: - location = reference.gcs_bucket.bucket - case storage_pb.STORAGE_TYPE_AWS_S3_BUCKET: - return S3StorageLocation( - id=uuid_message_to_uuid(storage_location.id), - location=reference.aws_s3_bucket.bucket, - type=StorageType.S3, - region=reference.aws_s3_bucket.region or None, - ) - case storage_pb.STORAGE_TYPE_FILESYSTEM: - location = reference.filesystem.path - return cls( - id=uuid_message_to_uuid(storage_location.id), - location=location, - type=_STORAGE_TYPE_TO_ENUM[storage_type], - ) + reference = storage_location.reference + match reference.type: + case storage_pb.STORAGE_TYPE_AZURE_BLOB: + return AzureBlobStorageLocation( + id=uuid_message_to_uuid(storage_location.id), + name=storage_location.name, + location=reference.azure_blob.container, + type=StorageType.AZURE_BLOB, + storage_account_resource_id=reference.azure_blob.storage_account_resource_id, + region=reference.azure_blob.region, + ) + case storage_pb.STORAGE_TYPE_GCS: + return GCSStorageLocation( + id=uuid_message_to_uuid(storage_location.id), + name=storage_location.name, + location=reference.gcs_bucket.bucket, + type=StorageType.GCS, + project_id=reference.gcs_bucket.project_id, + bucket_location=reference.gcs_bucket.location, + ) + case storage_pb.STORAGE_TYPE_AWS_S3: + return AWSS3StorageLocation( + id=uuid_message_to_uuid(storage_location.id), + name=storage_location.name, + location=reference.aws_s3_bucket.bucket, + type=StorageType.AWS_S3, + region=reference.aws_s3_bucket.region or None, + ) + case storage_pb.STORAGE_TYPE_LOCAL: + return cls( + id=uuid_message_to_uuid(storage_location.id), + name=storage_location.name, + location=reference.local.path, + type=StorageType.LOCAL, + ) + case _: + raise ValueError(f"Unsupported storage reference type: {reference.type}") def _with_runner_context(self, runner_context: "RunnerContext") -> "StorageLocation": return replace(self, runner_context=runner_context) def to_message(self) -> storage_pb.StorageLocation: """Convert a StorageLocation object to a StorageLocation protobuf message.""" - return storage_pb.StorageLocation( - id=uuid_to_uuid_message(self.id), location=self.location, type=self.type.value - ) + reference = storage_pb.StorageLocationReference(type=self.type.value) + match self.type: + case StorageType.AWS_S3: + reference.aws_s3_bucket.bucket = self.location + case StorageType.GCS: + reference.gcs_bucket.bucket = self.location + case StorageType.AZURE_BLOB: + reference.azure_blob.container = self.location + case StorageType.LOCAL: + reference.local.path = self.location + return storage_pb.StorageLocation(id=uuid_to_uuid_message(self.id), name=self.name, reference=reference) def read(self, path: str) -> bytes: runner_context = self.runner_context or RunnerContext() @@ -1042,21 +1124,26 @@ def read(self, path: str) -> bytes: case StorageType.GCS: span.set_attribute("bucket", self.location) store = runner_context.gcs_client(self.location) - case StorageType.S3: + case StorageType.AWS_S3: span.set_attribute("bucket", self.location) - if isinstance(self, S3StorageLocation) and self.region: + if isinstance(self, AWSS3StorageLocation) and self.region: store = runner_context.s3_client(self.location, region=self.region) else: store = runner_context.s3_client(self.location) - case StorageType.AZURE: - if not isinstance(self, AzureStorageLocation) or not self.storage_account_resource_id: + case StorageType.AZURE_BLOB: + if not isinstance(self, AzureBlobStorageLocation) or not self.storage_account_resource_id: raise ValueError("Azure storage location is missing its storage account resource ID") span.set_attribute("storage_account_resource_id", self.storage_account_resource_id) span.set_attribute("container", self.location) store = runner_context.azure_client(self.storage_account_resource_id, self.location) - case StorageType.FS: + case StorageType.LOCAL: + from obstore.store import LocalStore # noqa: PLC0415 + span.set_attribute("root_directory", self.location) - return Path(self.location).joinpath(path).read_bytes() + root = runner_context.local_path(self.location) + if not self.location or not root.is_dir(): + raise ValueError(f"Local storage location root is not available: {self.location!r}") + store = LocalStore(root) import obstore # noqa: PLC0415 @@ -1064,28 +1151,41 @@ def read(self, path: str) -> bytes: @dataclass(frozen=True) -class S3StorageLocation(StorageLocation): +class AWSS3StorageLocation(StorageLocation): region: str | None = field(default=None, kw_only=True) def to_message(self) -> storage_pb.StorageLocation: """Include the S3 bucket and its region in the storage reference.""" message = super().to_message() - message.reference.type = storage_pb.STORAGE_TYPE_AWS_S3_BUCKET - message.reference.aws_s3_bucket.bucket = self.location message.reference.aws_s3_bucket.region = self.region or "" return message @dataclass(frozen=True) -class AzureStorageLocation(StorageLocation): +class GCSStorageLocation(StorageLocation): + project_id: str = field(default="", kw_only=True) + bucket_location: str = field(default="", kw_only=True) + + def to_message(self) -> storage_pb.StorageLocation: + message = super().to_message() + message.reference.gcs_bucket.project_id = self.project_id + message.reference.gcs_bucket.location = self.bucket_location + return message + + +@dataclass(frozen=True) +class AzureBlobStorageLocation(StorageLocation): storage_account_resource_id: str = field(kw_only=True) + region: str = field(default="", kw_only=True) + + def __str__(self) -> str: + return f"{self.storage_account_resource_id}/containers/{self.location}" def to_message(self) -> storage_pb.StorageLocation: """Include the Azure account and container in the storage reference.""" message = super().to_message() - message.reference.type = storage_pb.STORAGE_TYPE_AZURE_BLOB message.reference.azure_blob.storage_account_resource_id = self.storage_account_resource_id - message.reference.azure_blob.container = self.location + message.reference.azure_blob.region = self.region return message diff --git a/tilebox-workflows/tilebox/workflows/runner/executor.py b/tilebox-workflows/tilebox/workflows/runner/executor.py index 7ef8a7e..fcef73e 100644 --- a/tilebox-workflows/tilebox/workflows/runner/executor.py +++ b/tilebox-workflows/tilebox/workflows/runner/executor.py @@ -257,7 +257,7 @@ def _load(self) -> None: return self._locations = { location.id: location._with_runner_context(self._runner_context) # noqa: SLF001 - for location in self._client.automations().storage_locations() + for location in self._client.storage_locations().all() } self._loaded = True diff --git a/tilebox-workflows/tilebox/workflows/storage_locations/__init__.py b/tilebox-workflows/tilebox/workflows/storage_locations/__init__.py new file mode 100644 index 0000000..1c823aa --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/storage_locations/__init__.py @@ -0,0 +1,35 @@ +from tilebox.workflows.data import ( + AWSS3BucketReference, + AWSS3StorageLocation, + AzureBlobReference, + AzureBlobStorageLocation, + GCSBucketReference, + GCSStorageLocation, + LocalReference, + StorageEventType, + StorageLocation, + StorageLocationReference, + StorageType, +) +from tilebox.workflows.storage_locations.client import StorageLocationClient +from tilebox.workflows.storage_locations.data import ( + StorageSubscriptionEvent, + TriggeredJob, +) + +__all__ = [ + "AWSS3BucketReference", + "AWSS3StorageLocation", + "AzureBlobReference", + "AzureBlobStorageLocation", + "GCSBucketReference", + "GCSStorageLocation", + "LocalReference", + "StorageEventType", + "StorageLocation", + "StorageLocationClient", + "StorageLocationReference", + "StorageSubscriptionEvent", + "StorageType", + "TriggeredJob", +] diff --git a/tilebox-workflows/tilebox/workflows/storage_locations/client.py b/tilebox-workflows/tilebox/workflows/storage_locations/client.py new file mode 100644 index 0000000..e592c64 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/storage_locations/client.py @@ -0,0 +1,38 @@ +from typing import TypeAlias +from uuid import UUID + +from tilebox.workflows.data import StorageLocation, StorageLocationReference +from tilebox.workflows.storage_locations.service import StorageLocationService + +StorageLocationIDLike: TypeAlias = StorageLocation | UUID | str + + +class StorageLocationClient: + def __init__(self, service: StorageLocationService) -> None: + self._service = service + + def all(self) -> list[StorageLocation]: + """List all available storage locations.""" + return self._service.list() + + def find(self, storage_location_id: StorageLocationIDLike) -> StorageLocation: + """Find a storage location by object, UUID, or UUID string.""" + return self._service.get_by_id(_to_uuid(storage_location_id)) + + def create(self, name: str, reference: StorageLocationReference) -> StorageLocation: + """Create a storage location from an AWS S3, GCS, Azure Blob, or local reference. + + Creating a location does not create the underlying bucket, container, or directory, + or configure an event subscription. + """ + return self._service.create(name, reference) + + def update(self, storage_location_id: StorageLocationIDLike, name: str) -> StorageLocation: + """Rename a storage location. Its storage reference cannot be changed.""" + return self._service.update(_to_uuid(storage_location_id), name) + + +def _to_uuid(location: StorageLocationIDLike) -> UUID: + if isinstance(location, StorageLocation): + return location.id + return UUID(location) if isinstance(location, str) else location diff --git a/tilebox-workflows/tilebox/workflows/storage_locations/data.py b/tilebox-workflows/tilebox/workflows/storage_locations/data.py new file mode 100644 index 0000000..d4a25d3 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/storage_locations/data.py @@ -0,0 +1,60 @@ +from dataclasses import dataclass +from datetime import datetime +from uuid import UUID + +from tilebox.datasets.query.pagination import Pagination +from tilebox.datasets.query.time_interval import timestamp_to_datetime +from tilebox.datasets.uuid import uuid_message_to_uuid +from tilebox.workflows.data import StorageEventType +from tilebox.workflows.workflows.v1 import storage_location_pb2 as storage_pb + + +@dataclass(frozen=True) +class TriggeredJob: + job_id: UUID + automation_id: UUID + glob_pattern: str + + @classmethod + def from_message(cls, job: storage_pb.TriggeredJob) -> "TriggeredJob": + return cls( + job_id=uuid_message_to_uuid(job.job_id), + automation_id=uuid_message_to_uuid(job.automation_id), + glob_pattern=job.glob_pattern, + ) + + +@dataclass(frozen=True) +class StorageSubscriptionEvent: + id: UUID + object_key: str + type: StorageEventType + event_time: datetime + received_at: datetime + triggered_jobs: list[TriggeredJob] + storage_subscription_id: UUID + + @classmethod + def from_message(cls, event: storage_pb.StorageSubscriptionEvent) -> "StorageSubscriptionEvent": + return cls( + id=uuid_message_to_uuid(event.id), + object_key=event.object_key, + type=StorageEventType(event.type), + event_time=timestamp_to_datetime(event.event_time), + received_at=timestamp_to_datetime(event.received_at), + triggered_jobs=[TriggeredJob.from_message(job) for job in event.triggered_jobs], + storage_subscription_id=uuid_message_to_uuid(event.storage_subscription_id), + ) + + +@dataclass(frozen=True) +class StorageSubscriptionEvents: + events: list[StorageSubscriptionEvent] + next_page: Pagination + + @classmethod + def from_message(cls, page: storage_pb.StorageSubscriptionEvents) -> "StorageSubscriptionEvents": + return cls( + events=[StorageSubscriptionEvent.from_message(event) for event in page.events], + next_page=Pagination.from_message(page.next_page), + ) diff --git a/tilebox-workflows/tilebox/workflows/storage_locations/service.py b/tilebox-workflows/tilebox/workflows/storage_locations/service.py new file mode 100644 index 0000000..61d4665 --- /dev/null +++ b/tilebox-workflows/tilebox/workflows/storage_locations/service.py @@ -0,0 +1,46 @@ +import builtins +from typing import Any +from uuid import UUID + +from google.protobuf.empty_pb2 import Empty +from grpc import Channel + +from _tilebox.grpc.error import with_pythonic_errors +from tilebox.datasets.query.pagination import Pagination +from tilebox.datasets.uuid import uuid_to_uuid_message +from tilebox.workflows.data import StorageLocation, StorageLocationReference +from tilebox.workflows.storage_locations.data import StorageSubscriptionEvents +from tilebox.workflows.workflows.v1.storage_location_pb2 import ( + CreateStorageLocationRequest, + ListStorageSubscriptionEventsRequest, + UpdateStorageLocationRequest, +) +from tilebox.workflows.workflows.v1.storage_location_pb2_grpc import StorageLocationServiceStub + + +class StorageLocationService: + def __init__(self, channel: Channel | Any) -> None: + self.service = ( + with_pythonic_errors(StorageLocationServiceStub(channel)) if hasattr(channel, "unary_unary") else channel + ) + + def list(self) -> builtins.list[StorageLocation]: + response = self.service.ListStorageLocations(Empty()) + return [StorageLocation.from_message(location) for location in response.locations] + + def get_by_id(self, storage_location_id: UUID) -> StorageLocation: + return StorageLocation.from_message(self.service.GetStorageLocation(uuid_to_uuid_message(storage_location_id))) + + def create(self, name: str, reference: StorageLocationReference) -> StorageLocation: + request = CreateStorageLocationRequest(name=name, reference=reference.to_message()) + return StorageLocation.from_message(self.service.CreateStorageLocation(request)) + + def update(self, storage_location_id: UUID, name: str) -> StorageLocation: + request = UpdateStorageLocationRequest(storage_location_id=uuid_to_uuid_message(storage_location_id), name=name) + return StorageLocation.from_message(self.service.UpdateStorageLocation(request)) + + def list_events(self, storage_location_id: UUID, page: Pagination) -> StorageSubscriptionEvents: + request = ListStorageSubscriptionEventsRequest( + storage_location_id=uuid_to_uuid_message(storage_location_id), page=page.to_message() + ) + return StorageSubscriptionEvents.from_message(self.service.ListStorageSubscriptionEvents(request)) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py index 6d76160..3930fd2 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/automation_pb2.py @@ -30,14 +30,14 @@ from tilebox.workflows.workflows.v1 import storage_location_pb2 as workflows_dot_v1_dot_storage__location__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\x1a#workflows/v1/storage_location.proto\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs2\xa4\x03\n\x11\x41utomationService\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflows/v1/automation.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\x1a#workflows/v1/storage_location.proto\"\xc0\x02\n\x13\x41utomationPrototype\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12@\n\tprototype\x18\x03 \x01(\x0b\x32\".workflows.v1.SingleTaskSubmissionR\tprototype\x12W\n\x16storage_event_triggers\x18\x04 \x03(\x0b\x32!.workflows.v1.StorageEventTriggerR\x14storageEventTriggers\x12>\n\rcron_triggers\x18\x05 \x03(\x0b\x32\x19.workflows.v1.CronTriggerR\x0c\x63ronTriggers\x12\x1a\n\x08\x64isabled\x18\x06 \x01(\x08R\x08\x64isabled\"R\n\x0b\x41utomations\x12\x43\n\x0b\x61utomations\x18\x01 \x03(\x0b\x32!.workflows.v1.AutomationPrototypeR\x0b\x61utomations\"\xa2\x01\n\x13StorageEventTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12H\n\x10storage_location\x18\x02 \x01(\x0b\x32\x1d.workflows.v1.StorageLocationR\x0fstorageLocation\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"\x96\x01\n\x0b\x43ronTrigger\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1a\n\x08schedule\x18\x02 \x01(\tR\x08schedule\x12K\n\x11next_scheduled_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\x0fnextScheduledAt\"E\n\nAutomation\x12#\n\rtrigger_event\x18\x01 \x01(\x0cR\x0ctriggerEvent\x12\x12\n\x04\x61rgs\x18\x02 \x01(\x0cR\x04\x61rgs\"S\n\x12TriggeredCronEvent\x12=\n\x0ctrigger_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0btriggerTime\"o\n\x17\x44\x65leteAutomationRequest\x12\x33\n\rautomation_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12\x1f\n\x0b\x63\x61ncel_jobs\x18\x02 \x01(\x08R\ncancelJobs2\xa4\x03\n\x11\x41utomationService\x12\x44\n\x0fListAutomations\x12\x16.google.protobuf.Empty\x1a\x19.workflows.v1.Automations\x12\x42\n\rGetAutomation\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10\x43reateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12X\n\x10UpdateAutomation\x12!.workflows.v1.AutomationPrototype\x1a!.workflows.v1.AutomationPrototype\x12Q\n\x10\x44\x65leteAutomation\x12%.workflows.v1.DeleteAutomationRequest\x1a\x16.google.protobuf.EmptyBy\n\x10\x63om.workflows.v1B\x0f\x41utomationProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.automation_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' + _globals['DESCRIPTOR']._serialized_options = b'\n\020com.workflows.v1B\017AutomationProtoP\001\242\002\003WXX\252\002\014Workflows.V1\312\002\014Workflows\\V1\342\002\030Workflows\\V1\\GPBMetadata\352\002\rWorkflows::V1\222\003\002\010\002' _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._loaded_options = None _globals['_CRONTRIGGER'].fields_by_name['next_scheduled_at']._serialized_options = b'\340A\003' _globals['_AUTOMATIONPROTOTYPE']._serialized_start=226 diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py index fcfb70e..b332b4b 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.py @@ -29,14 +29,14 @@ from tilebox.datasets.tilebox.v1 import query_pb2 as tilebox_dot_v1_dot_query__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#workflows/v1/storage_location.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xde\x01\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1e\n\x08location\x18\x02 \x01(\tB\x02\x18\x01R\x08location\x12\x31\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeB\x02\x18\x01R\x04type\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x44\n\treference\x18\x05 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"\xd6\x02\n\x18StorageLocationReference\x12\x46\n\raws_s3_bucket\x18\x01 \x01(\x0b\x32\".workflows.v1.AWSS3BucketReferenceR\x0b\x61wsS3Bucket\x12?\n\ngcs_bucket\x18\x02 \x01(\x0b\x32 .workflows.v1.GCSBucketReferenceR\tgcsBucket\x12?\n\nazure_blob\x18\x03 \x01(\x0b\x32 .workflows.v1.AzureBlobReferenceR\tazureBlob\x12\x41\n\nfilesystem\x18\x04 \x01(\x0b\x32!.workflows.v1.FilesystemReferenceR\nfilesystem\x12-\n\x04type\x18\x05 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"F\n\x14\x41WSS3BucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x16\n\x06region\x18\x02 \x01(\tR\x06region\"g\n\x12GCSBucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x1d\n\nproject_id\x18\x02 \x01(\tR\tprojectId\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"\x89\x01\n\x12\x41zureBlobReference\x12=\n\x1bstorage_account_resource_id\x18\x01 \x01(\tR\x18storageAccountResourceId\x12\x1c\n\tcontainer\x18\x02 \x01(\tR\tcontainer\x12\x16\n\x06region\x18\x03 \x01(\tR\x06region\")\n\x13\x46ilesystemReference\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\"\x8d\x01\n\x0cTriggeredJob\x12%\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05jobId\x12\x33\n\rautomation_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"R\n\rTriggeredJobs\x12\x41\n\x0etriggered_jobs\x18\x01 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\"x\n\x1c\x43reateStorageLocationRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\treference\x18\x02 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"r\n\x1cUpdateStorageLocationRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"8\n\x19\x41WSSNSStorageSubscription\x12\x1b\n\ttopic_arn\x18\x01 \x01(\tR\x08topicArn\"\x9a\x01\n\x1fGooglePubSubStorageSubscription\x12\"\n\x0csubscription\x18\x01 \x01(\tR\x0csubscription\x12\x32\n\x15service_account_email\x18\x02 \x01(\tR\x13serviceAccountEmail\x12\x1f\n\x08\x61udience\x18\x03 \x01(\tB\x03\xe0\x41\x03R\x08\x61udience\"\xbd\x01\n!AzureEventGridStorageSubscription\x12\x37\n\x15webhook_secret_header\x18\x01 \x01(\tB\x03\xe0\x41\x03R\x13webhookSecretHeader\x12*\n\x0ewebhook_secret\x18\x02 \x01(\tB\x03\xe0\x41\x03R\rwebhookSecret\x12\x33\n\x13webhook_secret_hash\x18\x03 \x01(\x0c\x42\x03\xe0\x41\x03R\x11webhookSecretHash\"\x87\x04\n\x13StorageSubscription\x12#\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x03\xe0\x41\x03R\x02id\x12>\n\x13storage_location_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x03 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12\x1f\n\x08\x65ndpoint\x18\x04 \x01(\tB\x03\xe0\x41\x03R\x08\x65ndpoint\x12>\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\tcreatedAt\x12@\n\x07\x61ws_sns\x18\x07 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x08 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\t \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"\x8e\x03\n CreateStorageSubscriptionRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x02 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12@\n\x07\x61ws_sns\x18\x03 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x04 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\x05 \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"a\n\x1fListStorageSubscriptionsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\"_\n\x14StorageSubscriptions\x12G\n\rsubscriptions\x18\x01 \x03(\x0b\x32!.workflows.v1.StorageSubscriptionR\rsubscriptions\"\x90\x03\n\x18StorageSubscriptionEvent\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1d\n\nobject_key\x18\x02 \x01(\tR\tobjectKey\x12\x32\n\x04type\x18\x03 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x39\n\nevent_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12;\n\x0breceived_at\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nreceivedAt\x12\x41\n\x0etriggered_jobs\x18\x06 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\x12\x46\n\x17storage_subscription_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x15storageSubscriptionId\"\x99\x01\n$ListStorageSubscriptionEventsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x97\x01\n\x19StorageSubscriptionEvents\x12>\n\x06\x65vents\x18\x01 \x03(\x0b\x32&.workflows.v1.StorageSubscriptionEventR\x06\x65vents\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location*\xa2\x01\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n\x17STORAGE_TYPE_GCS_BUCKET\x10\x01\x12\x1e\n\x1aSTORAGE_TYPE_AWS_S3_BUCKET\x10\x02\x12\x1b\n\x17STORAGE_TYPE_FILESYSTEM\x10\x03\x12\x1b\n\x17STORAGE_TYPE_AZURE_BLOB\x10\x04*\xc8\x01\n\x17StorageSubscriptionType\x12)\n%STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED\x10\x00\x12%\n!STORAGE_SUBSCRIPTION_TYPE_AWS_SNS\x10\x01\x12+\n\'STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB\x10\x02\x12.\n*STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID\x10\x03*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xa5\x07\n\x16StorageLocationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15\x43reateStorageLocation\x12*.workflows.v1.CreateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15UpdateStorageLocation\x12*.workflows.v1.UpdateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12n\n\x19\x43reateStorageSubscription\x12..workflows.v1.CreateStorageSubscriptionRequest\x1a!.workflows.v1.StorageSubscription\x12m\n\x18ListStorageSubscriptions\x12-.workflows.v1.ListStorageSubscriptionsRequest\x1a\".workflows.v1.StorageSubscriptions\x12K\n\x16GetStorageSubscription\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.StorageSubscription\x12\x43\n\x19\x44\x65leteStorageSubscription\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12|\n\x1dListStorageSubscriptionEvents\x12\x32.workflows.v1.ListStorageSubscriptionEventsRequest\x1a\'.workflows.v1.StorageSubscriptionEventsB\x05\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#workflows/v1/storage_location.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/api/field_behavior.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\"\xde\x01\n\x0fStorageLocation\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1e\n\x08location\x18\x02 \x01(\tB\x02\x18\x01R\x08location\x12\x31\n\x04type\x18\x03 \x01(\x0e\x32\x19.workflows.v1.StorageTypeB\x02\x18\x01R\x04type\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x44\n\treference\x18\x05 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"\xc7\x02\n\x18StorageLocationReference\x12\x46\n\raws_s3_bucket\x18\x01 \x01(\x0b\x32\".workflows.v1.AWSS3BucketReferenceR\x0b\x61wsS3Bucket\x12?\n\ngcs_bucket\x18\x02 \x01(\x0b\x32 .workflows.v1.GCSBucketReferenceR\tgcsBucket\x12?\n\nazure_blob\x18\x03 \x01(\x0b\x32 .workflows.v1.AzureBlobReferenceR\tazureBlob\x12\x32\n\x05local\x18\x04 \x01(\x0b\x32\x1c.workflows.v1.LocalReferenceR\x05local\x12-\n\x04type\x18\x05 \x01(\x0e\x32\x19.workflows.v1.StorageTypeR\x04type\"F\n\x14\x41WSS3BucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x16\n\x06region\x18\x02 \x01(\tR\x06region\"g\n\x12GCSBucketReference\x12\x16\n\x06\x62ucket\x18\x01 \x01(\tR\x06\x62ucket\x12\x1d\n\nproject_id\x18\x02 \x01(\tR\tprojectId\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location\"\x89\x01\n\x12\x41zureBlobReference\x12=\n\x1bstorage_account_resource_id\x18\x01 \x01(\tR\x18storageAccountResourceId\x12\x1c\n\tcontainer\x18\x02 \x01(\tR\tcontainer\x12\x16\n\x06region\x18\x03 \x01(\tR\x06region\"$\n\x0eLocalReference\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\"\x8d\x01\n\x0cTriggeredJob\x12%\n\x06job_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x05jobId\x12\x33\n\rautomation_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x61utomationId\x12!\n\x0cglob_pattern\x18\x03 \x01(\tR\x0bglobPattern\"R\n\rTriggeredJobs\x12\x41\n\x0etriggered_jobs\x18\x01 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\"x\n\x1c\x43reateStorageLocationRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\treference\x18\x02 \x01(\x0b\x32&.workflows.v1.StorageLocationReferenceR\treference\"r\n\x1cUpdateStorageLocationRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"_\n\x19\x41WSSNSStorageSubscription\x12\x1b\n\ttopic_arn\x18\x01 \x01(\tR\x08topicArn\x12%\n\x0emessage_format\x18\x02 \x01(\tR\rmessageFormat\"\x9a\x01\n\x1fGooglePubSubStorageSubscription\x12\"\n\x0csubscription\x18\x01 \x01(\tR\x0csubscription\x12\x32\n\x15service_account_email\x18\x02 \x01(\tR\x13serviceAccountEmail\x12\x1f\n\x08\x61udience\x18\x03 \x01(\tB\x03\xe0\x41\x03R\x08\x61udience\"\xbd\x01\n!AzureEventGridStorageSubscription\x12\x37\n\x15webhook_secret_header\x18\x01 \x01(\tB\x03\xe0\x41\x03R\x13webhookSecretHeader\x12*\n\x0ewebhook_secret\x18\x02 \x01(\tB\x03\xe0\x41\x03R\rwebhookSecret\x12\x33\n\x13webhook_secret_hash\x18\x03 \x01(\x0c\x42\x03\xe0\x41\x03R\x11webhookSecretHash\"\x87\x04\n\x13StorageSubscription\x12#\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDB\x03\xe0\x41\x03R\x02id\x12>\n\x13storage_location_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x03 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12\x1f\n\x08\x65ndpoint\x18\x04 \x01(\tB\x03\xe0\x41\x03R\x08\x65ndpoint\x12>\n\ncreated_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x03\xe0\x41\x03R\tcreatedAt\x12@\n\x07\x61ws_sns\x18\x07 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x08 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\t \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"\x8e\x03\n CreateStorageSubscriptionRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x39\n\x04type\x18\x02 \x01(\x0e\x32%.workflows.v1.StorageSubscriptionTypeR\x04type\x12@\n\x07\x61ws_sns\x18\x03 \x01(\x0b\x32\'.workflows.v1.AWSSNSStorageSubscriptionR\x06\x61wsSns\x12R\n\rgoogle_pubsub\x18\x04 \x01(\x0b\x32-.workflows.v1.GooglePubSubStorageSubscriptionR\x0cgooglePubsub\x12Y\n\x10\x61zure_event_grid\x18\x05 \x01(\x0b\x32/.workflows.v1.AzureEventGridStorageSubscriptionR\x0e\x61zureEventGrid\"a\n\x1fListStorageSubscriptionsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\"_\n\x14StorageSubscriptions\x12G\n\rsubscriptions\x18\x01 \x03(\x0b\x32!.workflows.v1.StorageSubscriptionR\rsubscriptions\"\x90\x03\n\x18StorageSubscriptionEvent\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x1d\n\nobject_key\x18\x02 \x01(\tR\tobjectKey\x12\x32\n\x04type\x18\x03 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x39\n\nevent_time\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12;\n\x0breceived_at\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\nreceivedAt\x12\x41\n\x0etriggered_jobs\x18\x06 \x03(\x0b\x32\x1a.workflows.v1.TriggeredJobR\rtriggeredJobs\x12\x46\n\x17storage_subscription_id\x18\x07 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x15storageSubscriptionId\"\x99\x01\n$ListStorageSubscriptionEventsRequest\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x31\n\x04page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x97\x01\n\x19StorageSubscriptionEvents\x12>\n\x06\x65vents\x18\x01 \x03(\x0b\x32&.workflows.v1.StorageSubscriptionEventR\x06\x65vents\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"O\n\x10StorageLocations\x12;\n\tlocations\x18\x01 \x03(\x0b\x32\x1d.workflows.v1.StorageLocationR\tlocations\"\xa7\x01\n\x15TriggeredStorageEvent\x12>\n\x13storage_location_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x11storageLocationId\x12\x32\n\x04type\x18\x02 \x01(\x0e\x32\x1e.workflows.v1.StorageEventTypeR\x04type\x12\x1a\n\x08location\x18\x03 \x01(\tR\x08location*\x8f\x01\n\x0bStorageType\x12\x1c\n\x18STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\x14\n\x10STORAGE_TYPE_GCS\x10\x01\x12\x17\n\x13STORAGE_TYPE_AWS_S3\x10\x02\x12\x16\n\x12STORAGE_TYPE_LOCAL\x10\x03\x12\x1b\n\x17STORAGE_TYPE_AZURE_BLOB\x10\x04*\xf3\x01\n\x17StorageSubscriptionType\x12)\n%STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED\x10\x00\x12%\n!STORAGE_SUBSCRIPTION_TYPE_AWS_SNS\x10\x01\x12+\n\'STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB\x10\x02\x12.\n*STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID\x10\x03\x12)\n%STORAGE_SUBSCRIPTION_TYPE_TILEBOX_CLI\x10\x04*V\n\x10StorageEventType\x12\"\n\x1eSTORAGE_EVENT_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTORAGE_EVENT_TYPE_CREATED\x10\x01\x32\xa5\x07\n\x16StorageLocationService\x12N\n\x14ListStorageLocations\x12\x16.google.protobuf.Empty\x1a\x1e.workflows.v1.StorageLocations\x12\x43\n\x12GetStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15\x43reateStorageLocation\x12*.workflows.v1.CreateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12\x62\n\x15UpdateStorageLocation\x12*.workflows.v1.UpdateStorageLocationRequest\x1a\x1d.workflows.v1.StorageLocation\x12?\n\x15\x44\x65leteStorageLocation\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12n\n\x19\x43reateStorageSubscription\x12..workflows.v1.CreateStorageSubscriptionRequest\x1a!.workflows.v1.StorageSubscription\x12m\n\x18ListStorageSubscriptions\x12-.workflows.v1.ListStorageSubscriptionsRequest\x1a\".workflows.v1.StorageSubscriptions\x12K\n\x16GetStorageSubscription\x12\x0e.tilebox.v1.ID\x1a!.workflows.v1.StorageSubscription\x12\x43\n\x19\x44\x65leteStorageSubscription\x12\x0e.tilebox.v1.ID\x1a\x16.google.protobuf.Empty\x12|\n\x1dListStorageSubscriptionEvents\x12\x32.workflows.v1.ListStorageSubscriptionEventsRequest\x1a\'.workflows.v1.StorageSubscriptionEventsB~\n\x10\x63om.workflows.v1B\x14StorageLocationProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflows.v1.storage_location_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'\222\003\002\010\002' + _globals['DESCRIPTOR']._serialized_options = b'\n\020com.workflows.v1B\024StorageLocationProtoP\001\242\002\003WXX\252\002\014Workflows.V1\312\002\014Workflows\\V1\342\002\030Workflows\\V1\\GPBMetadata\352\002\rWorkflows::V1\222\003\002\010\002' _globals['_STORAGELOCATION'].fields_by_name['location']._loaded_options = None _globals['_STORAGELOCATION'].fields_by_name['location']._serialized_options = b'\030\001' _globals['_STORAGELOCATION'].fields_by_name['type']._loaded_options = None @@ -59,56 +59,56 @@ _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._loaded_options = None _globals['_STORAGESUBSCRIPTIONEVENTS'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' - _globals['_STORAGETYPE']._serialized_start=4080 + _globals['_STORAGETYPE']._serialized_start=4099 _globals['_STORAGETYPE']._serialized_end=4242 _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_start=4245 - _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_end=4445 - _globals['_STORAGEEVENTTYPE']._serialized_start=4447 - _globals['_STORAGEEVENTTYPE']._serialized_end=4533 + _globals['_STORAGESUBSCRIPTIONTYPE']._serialized_end=4488 + _globals['_STORAGEEVENTTYPE']._serialized_start=4490 + _globals['_STORAGEEVENTTYPE']._serialized_end=4576 _globals['_STORAGELOCATION']._serialized_start=194 _globals['_STORAGELOCATION']._serialized_end=416 _globals['_STORAGELOCATIONREFERENCE']._serialized_start=419 - _globals['_STORAGELOCATIONREFERENCE']._serialized_end=761 - _globals['_AWSS3BUCKETREFERENCE']._serialized_start=763 - _globals['_AWSS3BUCKETREFERENCE']._serialized_end=833 - _globals['_GCSBUCKETREFERENCE']._serialized_start=835 - _globals['_GCSBUCKETREFERENCE']._serialized_end=938 - _globals['_AZUREBLOBREFERENCE']._serialized_start=941 - _globals['_AZUREBLOBREFERENCE']._serialized_end=1078 - _globals['_FILESYSTEMREFERENCE']._serialized_start=1080 - _globals['_FILESYSTEMREFERENCE']._serialized_end=1121 - _globals['_TRIGGEREDJOB']._serialized_start=1124 - _globals['_TRIGGEREDJOB']._serialized_end=1265 - _globals['_TRIGGEREDJOBS']._serialized_start=1267 - _globals['_TRIGGEREDJOBS']._serialized_end=1349 - _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_start=1351 - _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_end=1471 - _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_start=1473 - _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_end=1587 - _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_start=1589 - _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_end=1645 - _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_start=1648 - _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_end=1802 - _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_start=1805 - _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_end=1994 - _globals['_STORAGESUBSCRIPTION']._serialized_start=1997 - _globals['_STORAGESUBSCRIPTION']._serialized_end=2516 - _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_start=2519 - _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_end=2917 - _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_start=2919 - _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_end=3016 - _globals['_STORAGESUBSCRIPTIONS']._serialized_start=3018 - _globals['_STORAGESUBSCRIPTIONS']._serialized_end=3113 - _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_start=3116 - _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_end=3516 - _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_start=3519 - _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_end=3672 - _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_start=3675 - _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_end=3826 - _globals['_STORAGELOCATIONS']._serialized_start=3828 - _globals['_STORAGELOCATIONS']._serialized_end=3907 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=3910 - _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=4077 - _globals['_STORAGELOCATIONSERVICE']._serialized_start=4536 - _globals['_STORAGELOCATIONSERVICE']._serialized_end=5469 + _globals['_STORAGELOCATIONREFERENCE']._serialized_end=746 + _globals['_AWSS3BUCKETREFERENCE']._serialized_start=748 + _globals['_AWSS3BUCKETREFERENCE']._serialized_end=818 + _globals['_GCSBUCKETREFERENCE']._serialized_start=820 + _globals['_GCSBUCKETREFERENCE']._serialized_end=923 + _globals['_AZUREBLOBREFERENCE']._serialized_start=926 + _globals['_AZUREBLOBREFERENCE']._serialized_end=1063 + _globals['_LOCALREFERENCE']._serialized_start=1065 + _globals['_LOCALREFERENCE']._serialized_end=1101 + _globals['_TRIGGEREDJOB']._serialized_start=1104 + _globals['_TRIGGEREDJOB']._serialized_end=1245 + _globals['_TRIGGEREDJOBS']._serialized_start=1247 + _globals['_TRIGGEREDJOBS']._serialized_end=1329 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_start=1331 + _globals['_CREATESTORAGELOCATIONREQUEST']._serialized_end=1451 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_start=1453 + _globals['_UPDATESTORAGELOCATIONREQUEST']._serialized_end=1567 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_start=1569 + _globals['_AWSSNSSTORAGESUBSCRIPTION']._serialized_end=1664 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_start=1667 + _globals['_GOOGLEPUBSUBSTORAGESUBSCRIPTION']._serialized_end=1821 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_start=1824 + _globals['_AZUREEVENTGRIDSTORAGESUBSCRIPTION']._serialized_end=2013 + _globals['_STORAGESUBSCRIPTION']._serialized_start=2016 + _globals['_STORAGESUBSCRIPTION']._serialized_end=2535 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_start=2538 + _globals['_CREATESTORAGESUBSCRIPTIONREQUEST']._serialized_end=2936 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_start=2938 + _globals['_LISTSTORAGESUBSCRIPTIONSREQUEST']._serialized_end=3035 + _globals['_STORAGESUBSCRIPTIONS']._serialized_start=3037 + _globals['_STORAGESUBSCRIPTIONS']._serialized_end=3132 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_start=3135 + _globals['_STORAGESUBSCRIPTIONEVENT']._serialized_end=3535 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_start=3538 + _globals['_LISTSTORAGESUBSCRIPTIONEVENTSREQUEST']._serialized_end=3691 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_start=3694 + _globals['_STORAGESUBSCRIPTIONEVENTS']._serialized_end=3845 + _globals['_STORAGELOCATIONS']._serialized_start=3847 + _globals['_STORAGELOCATIONS']._serialized_end=3926 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_start=3929 + _globals['_TRIGGEREDSTORAGEEVENT']._serialized_end=4096 + _globals['_STORAGELOCATIONSERVICE']._serialized_start=4579 + _globals['_STORAGELOCATIONSERVICE']._serialized_end=5512 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi index 3e46926..9ea9a90 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/storage_location_pb2.pyi @@ -15,9 +15,9 @@ DESCRIPTOR: _descriptor.FileDescriptor class StorageType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () STORAGE_TYPE_UNSPECIFIED: _ClassVar[StorageType] - STORAGE_TYPE_GCS_BUCKET: _ClassVar[StorageType] - STORAGE_TYPE_AWS_S3_BUCKET: _ClassVar[StorageType] - STORAGE_TYPE_FILESYSTEM: _ClassVar[StorageType] + STORAGE_TYPE_GCS: _ClassVar[StorageType] + STORAGE_TYPE_AWS_S3: _ClassVar[StorageType] + STORAGE_TYPE_LOCAL: _ClassVar[StorageType] STORAGE_TYPE_AZURE_BLOB: _ClassVar[StorageType] class StorageSubscriptionType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): @@ -26,20 +26,22 @@ class StorageSubscriptionType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper) STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: _ClassVar[StorageSubscriptionType] STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: _ClassVar[StorageSubscriptionType] STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: _ClassVar[StorageSubscriptionType] + STORAGE_SUBSCRIPTION_TYPE_TILEBOX_CLI: _ClassVar[StorageSubscriptionType] class StorageEventType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () STORAGE_EVENT_TYPE_UNSPECIFIED: _ClassVar[StorageEventType] STORAGE_EVENT_TYPE_CREATED: _ClassVar[StorageEventType] STORAGE_TYPE_UNSPECIFIED: StorageType -STORAGE_TYPE_GCS_BUCKET: StorageType -STORAGE_TYPE_AWS_S3_BUCKET: StorageType -STORAGE_TYPE_FILESYSTEM: StorageType +STORAGE_TYPE_GCS: StorageType +STORAGE_TYPE_AWS_S3: StorageType +STORAGE_TYPE_LOCAL: StorageType STORAGE_TYPE_AZURE_BLOB: StorageType STORAGE_SUBSCRIPTION_TYPE_UNSPECIFIED: StorageSubscriptionType STORAGE_SUBSCRIPTION_TYPE_AWS_SNS: StorageSubscriptionType STORAGE_SUBSCRIPTION_TYPE_GOOGLE_PUBSUB: StorageSubscriptionType STORAGE_SUBSCRIPTION_TYPE_AZURE_EVENT_GRID: StorageSubscriptionType +STORAGE_SUBSCRIPTION_TYPE_TILEBOX_CLI: StorageSubscriptionType STORAGE_EVENT_TYPE_UNSPECIFIED: StorageEventType STORAGE_EVENT_TYPE_CREATED: StorageEventType @@ -58,18 +60,18 @@ class StorageLocation(_message.Message): def __init__(self, id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., location: _Optional[str] = ..., type: _Optional[_Union[StorageType, str]] = ..., name: _Optional[str] = ..., reference: _Optional[_Union[StorageLocationReference, _Mapping]] = ...) -> None: ... class StorageLocationReference(_message.Message): - __slots__ = ("aws_s3_bucket", "gcs_bucket", "azure_blob", "filesystem", "type") + __slots__ = ("aws_s3_bucket", "gcs_bucket", "azure_blob", "local", "type") AWS_S3_BUCKET_FIELD_NUMBER: _ClassVar[int] GCS_BUCKET_FIELD_NUMBER: _ClassVar[int] AZURE_BLOB_FIELD_NUMBER: _ClassVar[int] - FILESYSTEM_FIELD_NUMBER: _ClassVar[int] + LOCAL_FIELD_NUMBER: _ClassVar[int] TYPE_FIELD_NUMBER: _ClassVar[int] aws_s3_bucket: AWSS3BucketReference gcs_bucket: GCSBucketReference azure_blob: AzureBlobReference - filesystem: FilesystemReference + local: LocalReference type: StorageType - def __init__(self, aws_s3_bucket: _Optional[_Union[AWSS3BucketReference, _Mapping]] = ..., gcs_bucket: _Optional[_Union[GCSBucketReference, _Mapping]] = ..., azure_blob: _Optional[_Union[AzureBlobReference, _Mapping]] = ..., filesystem: _Optional[_Union[FilesystemReference, _Mapping]] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... + def __init__(self, aws_s3_bucket: _Optional[_Union[AWSS3BucketReference, _Mapping]] = ..., gcs_bucket: _Optional[_Union[GCSBucketReference, _Mapping]] = ..., azure_blob: _Optional[_Union[AzureBlobReference, _Mapping]] = ..., local: _Optional[_Union[LocalReference, _Mapping]] = ..., type: _Optional[_Union[StorageType, str]] = ...) -> None: ... class AWSS3BucketReference(_message.Message): __slots__ = ("bucket", "region") @@ -99,7 +101,7 @@ class AzureBlobReference(_message.Message): region: str def __init__(self, storage_account_resource_id: _Optional[str] = ..., container: _Optional[str] = ..., region: _Optional[str] = ...) -> None: ... -class FilesystemReference(_message.Message): +class LocalReference(_message.Message): __slots__ = ("path",) PATH_FIELD_NUMBER: _ClassVar[int] path: str @@ -138,10 +140,12 @@ class UpdateStorageLocationRequest(_message.Message): def __init__(self, storage_location_id: _Optional[_Union[_id_pb2.ID, _Mapping]] = ..., name: _Optional[str] = ...) -> None: ... class AWSSNSStorageSubscription(_message.Message): - __slots__ = ("topic_arn",) + __slots__ = ("topic_arn", "message_format") TOPIC_ARN_FIELD_NUMBER: _ClassVar[int] + MESSAGE_FORMAT_FIELD_NUMBER: _ClassVar[int] topic_arn: str - def __init__(self, topic_arn: _Optional[str] = ...) -> None: ... + message_format: str + def __init__(self, topic_arn: _Optional[str] = ..., message_format: _Optional[str] = ...) -> None: ... class GooglePubSubStorageSubscription(_message.Message): __slots__ = ("subscription", "service_account_email", "audience") diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_connect.py b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_connect.py index e8ecbaa..5262cda 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_connect.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_connect.py @@ -39,6 +39,9 @@ async def create_workflow(self, request: workflows_dot_v1_dot_workflows__pb2.Cre async def list_workflows(self, request: workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + async def list_public_workflows(self, request: workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + async def get_workflow(self, request: workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.Workflow: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") @@ -136,6 +139,16 @@ def __init__(self, service: WorkflowsService | AsyncGenerator[WorkflowsService], ), function=svc.list_workflows, ), + "/workflows.v1.WorkflowsService/ListPublicWorkflows": Endpoint.unary( + method=MethodInfo( + name="ListPublicWorkflows", + service_name="workflows.v1.WorkflowsService", + input=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + output=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=svc.list_public_workflows, + ), "/workflows.v1.WorkflowsService/GetWorkflow": Endpoint.unary( method=MethodInfo( name="GetWorkflow", @@ -360,6 +373,26 @@ async def list_workflows( timeout_ms=timeout_ms, ) + async def list_public_workflows( + self, + request: workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse: + return await self.execute_unary( + request=request, + method=MethodInfo( + name="ListPublicWorkflows", + service_name="workflows.v1.WorkflowsService", + input=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + output=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + async def get_workflow( self, request: workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest, @@ -519,6 +552,8 @@ def create_workflow(self, request: workflows_dot_v1_dot_workflows__pb2.CreateWor raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def list_workflows(self, request: workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") + def list_public_workflows(self, request: workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse: + raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def get_workflow(self, request: workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.Workflow: raise ConnectError(Code.UNIMPLEMENTED, "Not implemented") def update_workflow(self, request: workflows_dot_v1_dot_workflows__pb2.UpdateWorkflowRequest, ctx: RequestContext) -> workflows_dot_v1_dot_workflows__pb2.Workflow: @@ -609,6 +644,16 @@ def __init__(self, service: WorkflowsServiceSync, interceptors: Iterable[Interce ), function=service.list_workflows, ), + "/workflows.v1.WorkflowsService/ListPublicWorkflows": EndpointSync.unary( + method=MethodInfo( + name="ListPublicWorkflows", + service_name="workflows.v1.WorkflowsService", + input=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + output=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + function=service.list_public_workflows, + ), "/workflows.v1.WorkflowsService/GetWorkflow": EndpointSync.unary( method=MethodInfo( name="GetWorkflow", @@ -833,6 +878,26 @@ def list_workflows( timeout_ms=timeout_ms, ) + def list_public_workflows( + self, + request: workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + *, + headers: Headers | Mapping[str, str] | None = None, + timeout_ms: int | None = None, + ) -> workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse: + return self.execute_unary( + request=request, + method=MethodInfo( + name="ListPublicWorkflows", + service_name="workflows.v1.WorkflowsService", + input=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest, + output=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse, + idempotency_level=IdempotencyLevel.UNKNOWN, + ), + headers=headers, + timeout_ms=timeout_ms, + ) + def get_workflow( self, request: workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest, diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py index 6ab26f2..f09b11f 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.py @@ -24,10 +24,11 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as tilebox_dot_v1_dot_query__pb2 from tilebox.workflows.workflows.v1 import core_pb2 as workflows_dot_v1_dot_core__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/workflows.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x17workflows/v1/core.proto\"\xcb\x01\n\x07\x43luster\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12!\n\x0c\x64isplay_name\x18\x03 \x01(\tR\x0b\x64isplayName\x12 \n\x0b\x64\x65scription\x18\x06 \x01(\tR\x0b\x64\x65scription\x12\x1c\n\tdeletable\x18\x04 \x01(\x08R\tdeletable\x12\x43\n\x11\x64\x65ployed_releases\x18\x05 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\x10\x64\x65ployedReleasesJ\x04\x08\x01\x10\x02\"`\n\x14\x43reateClusterRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12\x12\n\x04slug\x18\x03 \x01(\tR\x04slug\"6\n\x11GetClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"}\n\x14UpdateClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\x12\x19\n\x04name\x18\x02 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x04name\x12\'\n\x0b\x64\x65scription\x18\x03 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x0b\x64\x65scription\"9\n\x14\x44\x65leteClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"\x17\n\x15\x44\x65leteClusterResponse\"\x15\n\x13ListClustersRequest\"I\n\x14ListClustersResponse\x12\x31\n\x08\x63lusters\x18\x01 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"9\n\x12GetWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\"\x80\x01\n\x15UpdateWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12\x19\n\x04name\x18\x02 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x04name\x12\'\n\x0b\x64\x65scription\x18\x03 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x0b\x64\x65scription\"<\n\x15\x44\x65leteWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\"\x18\n\x16\x44\x65leteWorkflowResponse\"\xad\x01\n\x1dPublishWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12/\n\x0b\x61rtifact_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\nartifactId\x12\x36\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentR\x07\x63ontent\"u\n\x1fUnpublishWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\"\"\n UnpublishWorkflowReleaseResponse\"\x16\n\x14ListWorkflowsRequest\"M\n\x15ListWorkflowsResponse\x12\x34\n\tworkflows\x18\x01 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\tworkflows\"M\n\x15\x43reateWorkflowRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\"\x95\x01\n\x08Workflow\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x04 \x01(\tR\x0b\x64\x65scription\x12\x39\n\x08releases\x18\x05 \x03(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x08releasesJ\x04\x08\x01\x10\x02\"\x8b\x02\n\x0fWorkflowRelease\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x32\n\x08\x61rtifact\x18\x02 \x01(\x0b\x32\x16.workflows.v1.ArtifactR\x08\x61rtifact\x12\x36\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentR\x07\x63ontent\x12\x39\n\ncreated_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x31\n\x08\x63lusters\x18\x05 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"B\n\x08\x41rtifact\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x16\n\x06\x64igest\x18\x02 \x01(\tR\x06\x64igest\"\xe9\x01\n\x0eReleaseContent\x12 \n\x0b\x66ingerprint\x18\x01 \x01(\tR\x0b\x66ingerprint\x12\x32\n\x05tasks\x18\x02 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\x05tasks\x12(\n\x05\x66iles\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x05\x66iles\x12,\n\x12runner_object_path\x18\x04 \x01(\tR\x10runnerObjectPath\x12)\n\x10\x63ommand_override\x18\x05 \x03(\tR\x0f\x63ommandOverride\"h\n\x04Path\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\x12\x1c\n\tdirectory\x18\x02 \x01(\x08R\tdirectory\x12.\n\x08\x63hildren\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x08\x63hildren\"\x97\x01\n\x1c\x44\x65ployWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12#\n\rcluster_slugs\x18\x03 \x03(\tR\x0c\x63lusterSlugs\"\x8b\x01\n\x1d\x44\x65ployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"\x99\x01\n\x1eUndeployWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12#\n\rcluster_slugs\x18\x03 \x03(\tR\x0c\x63lusterSlugs\"\x8d\x01\n\x1fUndeployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters2\x8a\n\n\x10WorkflowsService\x12J\n\rCreateCluster\x12\".workflows.v1.CreateClusterRequest\x1a\x15.workflows.v1.Cluster\x12\x44\n\nGetCluster\x12\x1f.workflows.v1.GetClusterRequest\x1a\x15.workflows.v1.Cluster\x12J\n\rUpdateCluster\x12\".workflows.v1.UpdateClusterRequest\x1a\x15.workflows.v1.Cluster\x12X\n\rDeleteCluster\x12\".workflows.v1.DeleteClusterRequest\x1a#.workflows.v1.DeleteClusterResponse\x12U\n\x0cListClusters\x12!.workflows.v1.ListClustersRequest\x1a\".workflows.v1.ListClustersResponse\x12M\n\x0e\x43reateWorkflow\x12#.workflows.v1.CreateWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12X\n\rListWorkflows\x12\".workflows.v1.ListWorkflowsRequest\x1a#.workflows.v1.ListWorkflowsResponse\x12G\n\x0bGetWorkflow\x12 .workflows.v1.GetWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12M\n\x0eUpdateWorkflow\x12#.workflows.v1.UpdateWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12[\n\x0e\x44\x65leteWorkflow\x12#.workflows.v1.DeleteWorkflowRequest\x1a$.workflows.v1.DeleteWorkflowResponse\x12\x64\n\x16PublishWorkflowRelease\x12+.workflows.v1.PublishWorkflowReleaseRequest\x1a\x1d.workflows.v1.WorkflowRelease\x12y\n\x18UnpublishWorkflowRelease\x12-.workflows.v1.UnpublishWorkflowReleaseRequest\x1a..workflows.v1.UnpublishWorkflowReleaseResponse\x12p\n\x15\x44\x65ployWorkflowRelease\x12*.workflows.v1.DeployWorkflowReleaseRequest\x1a+.workflows.v1.DeployWorkflowReleaseResponse\x12v\n\x17UndeployWorkflowRelease\x12,.workflows.v1.UndeployWorkflowReleaseRequest\x1a-.workflows.v1.UndeployWorkflowReleaseResponseBx\n\x10\x63om.workflows.v1B\x0eWorkflowsProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cworkflows/v1/workflows.proto\x12\x0cworkflows.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x13tilebox/v1/id.proto\x1a\x16tilebox/v1/query.proto\x1a\x17workflows/v1/core.proto\"\xcb\x01\n\x07\x43luster\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12!\n\x0c\x64isplay_name\x18\x03 \x01(\tR\x0b\x64isplayName\x12 \n\x0b\x64\x65scription\x18\x06 \x01(\tR\x0b\x64\x65scription\x12\x1c\n\tdeletable\x18\x04 \x01(\x08R\tdeletable\x12\x43\n\x11\x64\x65ployed_releases\x18\x05 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\x10\x64\x65ployedReleasesJ\x04\x08\x01\x10\x02\"`\n\x14\x43reateClusterRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\x12\x12\n\x04slug\x18\x03 \x01(\tR\x04slug\"6\n\x11GetClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"}\n\x14UpdateClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\x12\x19\n\x04name\x18\x02 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x04name\x12\'\n\x0b\x64\x65scription\x18\x03 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x0b\x64\x65scription\"9\n\x14\x44\x65leteClusterRequest\x12!\n\x0c\x63luster_slug\x18\x01 \x01(\tR\x0b\x63lusterSlug\"\x17\n\x15\x44\x65leteClusterResponse\"\x15\n\x13ListClustersRequest\"I\n\x14ListClustersResponse\x12\x31\n\x08\x63lusters\x18\x01 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"9\n\x12GetWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\"\x80\x01\n\x15UpdateWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12\x19\n\x04name\x18\x02 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x04name\x12\'\n\x0b\x64\x65scription\x18\x03 \x01(\tB\x05\xaa\x01\x02\x08\x01R\x0b\x64\x65scription\"<\n\x15\x44\x65leteWorkflowRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\"\x18\n\x16\x44\x65leteWorkflowResponse\"\xad\x01\n\x1dPublishWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12/\n\x0b\x61rtifact_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\nartifactId\x12\x36\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentR\x07\x63ontent\"u\n\x1fUnpublishWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\"\"\n UnpublishWorkflowReleaseResponse\"I\n\x14ListWorkflowsRequest\x12\x31\n\x04page\x18\x01 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x89\x01\n\x15ListWorkflowsResponse\x12\x34\n\tworkflows\x18\x01 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\tworkflows\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"O\n\x1aListPublicWorkflowsRequest\x12\x31\n\x04page\x18\x01 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x04page\"\x8f\x01\n\x1bListPublicWorkflowsResponse\x12\x34\n\tworkflows\x18\x01 \x03(\x0b\x32\x16.workflows.v1.WorkflowR\tworkflows\x12:\n\tnext_page\x18\x02 \x01(\x0b\x32\x16.tilebox.v1.PaginationB\x05\xaa\x01\x02\x08\x01R\x08nextPage\"M\n\x15\x43reateWorkflowRequest\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x02 \x01(\tR\x0b\x64\x65scription\"\x95\x01\n\x08Workflow\x12\x12\n\x04slug\x18\x02 \x01(\tR\x04slug\x12\x12\n\x04name\x18\x03 \x01(\tR\x04name\x12 \n\x0b\x64\x65scription\x18\x04 \x01(\tR\x0b\x64\x65scription\x12\x39\n\x08releases\x18\x05 \x03(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x08releasesJ\x04\x08\x01\x10\x02\"\x8b\x02\n\x0fWorkflowRelease\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x32\n\x08\x61rtifact\x18\x02 \x01(\x0b\x32\x16.workflows.v1.ArtifactR\x08\x61rtifact\x12\x36\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1c.workflows.v1.ReleaseContentR\x07\x63ontent\x12\x39\n\ncreated_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x31\n\x08\x63lusters\x18\x05 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"B\n\x08\x41rtifact\x12\x1e\n\x02id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x02id\x12\x16\n\x06\x64igest\x18\x02 \x01(\tR\x06\x64igest\"\xe9\x01\n\x0eReleaseContent\x12 \n\x0b\x66ingerprint\x18\x01 \x01(\tR\x0b\x66ingerprint\x12\x32\n\x05tasks\x18\x02 \x03(\x0b\x32\x1c.workflows.v1.TaskIdentifierR\x05tasks\x12(\n\x05\x66iles\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x05\x66iles\x12,\n\x12runner_object_path\x18\x04 \x01(\tR\x10runnerObjectPath\x12)\n\x10\x63ommand_override\x18\x05 \x03(\tR\x0f\x63ommandOverride\"h\n\x04Path\x12\x12\n\x04path\x18\x01 \x01(\tR\x04path\x12\x1c\n\tdirectory\x18\x02 \x01(\x08R\tdirectory\x12.\n\x08\x63hildren\x18\x03 \x03(\x0b\x32\x12.workflows.v1.PathR\x08\x63hildren\"\x97\x01\n\x1c\x44\x65ployWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12#\n\rcluster_slugs\x18\x03 \x03(\tR\x0c\x63lusterSlugs\"\x8b\x01\n\x1d\x44\x65ployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters\"\x99\x01\n\x1eUndeployWorkflowReleaseRequest\x12#\n\rworkflow_slug\x18\x01 \x01(\tR\x0cworkflowSlug\x12-\n\nrelease_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\treleaseId\x12#\n\rcluster_slugs\x18\x03 \x03(\tR\x0c\x63lusterSlugs\"\x8d\x01\n\x1fUndeployWorkflowReleaseResponse\x12\x37\n\x07release\x18\x01 \x01(\x0b\x32\x1d.workflows.v1.WorkflowReleaseR\x07release\x12\x31\n\x08\x63lusters\x18\x02 \x03(\x0b\x32\x15.workflows.v1.ClusterR\x08\x63lusters2\xf6\n\n\x10WorkflowsService\x12J\n\rCreateCluster\x12\".workflows.v1.CreateClusterRequest\x1a\x15.workflows.v1.Cluster\x12\x44\n\nGetCluster\x12\x1f.workflows.v1.GetClusterRequest\x1a\x15.workflows.v1.Cluster\x12J\n\rUpdateCluster\x12\".workflows.v1.UpdateClusterRequest\x1a\x15.workflows.v1.Cluster\x12X\n\rDeleteCluster\x12\".workflows.v1.DeleteClusterRequest\x1a#.workflows.v1.DeleteClusterResponse\x12U\n\x0cListClusters\x12!.workflows.v1.ListClustersRequest\x1a\".workflows.v1.ListClustersResponse\x12M\n\x0e\x43reateWorkflow\x12#.workflows.v1.CreateWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12X\n\rListWorkflows\x12\".workflows.v1.ListWorkflowsRequest\x1a#.workflows.v1.ListWorkflowsResponse\x12j\n\x13ListPublicWorkflows\x12(.workflows.v1.ListPublicWorkflowsRequest\x1a).workflows.v1.ListPublicWorkflowsResponse\x12G\n\x0bGetWorkflow\x12 .workflows.v1.GetWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12M\n\x0eUpdateWorkflow\x12#.workflows.v1.UpdateWorkflowRequest\x1a\x16.workflows.v1.Workflow\x12[\n\x0e\x44\x65leteWorkflow\x12#.workflows.v1.DeleteWorkflowRequest\x1a$.workflows.v1.DeleteWorkflowResponse\x12\x64\n\x16PublishWorkflowRelease\x12+.workflows.v1.PublishWorkflowReleaseRequest\x1a\x1d.workflows.v1.WorkflowRelease\x12y\n\x18UnpublishWorkflowRelease\x12-.workflows.v1.UnpublishWorkflowReleaseRequest\x1a..workflows.v1.UnpublishWorkflowReleaseResponse\x12p\n\x15\x44\x65ployWorkflowRelease\x12*.workflows.v1.DeployWorkflowReleaseRequest\x1a+.workflows.v1.DeployWorkflowReleaseResponse\x12v\n\x17UndeployWorkflowRelease\x12,.workflows.v1.UndeployWorkflowReleaseRequest\x1a-.workflows.v1.UndeployWorkflowReleaseResponseBx\n\x10\x63om.workflows.v1B\x0eWorkflowsProtoP\x01\xa2\x02\x03WXX\xaa\x02\x0cWorkflows.V1\xca\x02\x0cWorkflows\\V1\xe2\x02\x18Workflows\\V1\\GPBMetadata\xea\x02\rWorkflows::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -43,60 +44,72 @@ _globals['_UPDATEWORKFLOWREQUEST'].fields_by_name['name']._serialized_options = b'\252\001\002\010\001' _globals['_UPDATEWORKFLOWREQUEST'].fields_by_name['description']._loaded_options = None _globals['_UPDATEWORKFLOWREQUEST'].fields_by_name['description']._serialized_options = b'\252\001\002\010\001' - _globals['_CLUSTER']._serialized_start=126 - _globals['_CLUSTER']._serialized_end=329 - _globals['_CREATECLUSTERREQUEST']._serialized_start=331 - _globals['_CREATECLUSTERREQUEST']._serialized_end=427 - _globals['_GETCLUSTERREQUEST']._serialized_start=429 - _globals['_GETCLUSTERREQUEST']._serialized_end=483 - _globals['_UPDATECLUSTERREQUEST']._serialized_start=485 - _globals['_UPDATECLUSTERREQUEST']._serialized_end=610 - _globals['_DELETECLUSTERREQUEST']._serialized_start=612 - _globals['_DELETECLUSTERREQUEST']._serialized_end=669 - _globals['_DELETECLUSTERRESPONSE']._serialized_start=671 - _globals['_DELETECLUSTERRESPONSE']._serialized_end=694 - _globals['_LISTCLUSTERSREQUEST']._serialized_start=696 - _globals['_LISTCLUSTERSREQUEST']._serialized_end=717 - _globals['_LISTCLUSTERSRESPONSE']._serialized_start=719 - _globals['_LISTCLUSTERSRESPONSE']._serialized_end=792 - _globals['_GETWORKFLOWREQUEST']._serialized_start=794 - _globals['_GETWORKFLOWREQUEST']._serialized_end=851 - _globals['_UPDATEWORKFLOWREQUEST']._serialized_start=854 - _globals['_UPDATEWORKFLOWREQUEST']._serialized_end=982 - _globals['_DELETEWORKFLOWREQUEST']._serialized_start=984 - _globals['_DELETEWORKFLOWREQUEST']._serialized_end=1044 - _globals['_DELETEWORKFLOWRESPONSE']._serialized_start=1046 - _globals['_DELETEWORKFLOWRESPONSE']._serialized_end=1070 - _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_start=1073 - _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_end=1246 - _globals['_UNPUBLISHWORKFLOWRELEASEREQUEST']._serialized_start=1248 - _globals['_UNPUBLISHWORKFLOWRELEASEREQUEST']._serialized_end=1365 - _globals['_UNPUBLISHWORKFLOWRELEASERESPONSE']._serialized_start=1367 - _globals['_UNPUBLISHWORKFLOWRELEASERESPONSE']._serialized_end=1401 - _globals['_LISTWORKFLOWSREQUEST']._serialized_start=1403 - _globals['_LISTWORKFLOWSREQUEST']._serialized_end=1425 - _globals['_LISTWORKFLOWSRESPONSE']._serialized_start=1427 - _globals['_LISTWORKFLOWSRESPONSE']._serialized_end=1504 - _globals['_CREATEWORKFLOWREQUEST']._serialized_start=1506 - _globals['_CREATEWORKFLOWREQUEST']._serialized_end=1583 - _globals['_WORKFLOW']._serialized_start=1586 - _globals['_WORKFLOW']._serialized_end=1735 - _globals['_WORKFLOWRELEASE']._serialized_start=1738 - _globals['_WORKFLOWRELEASE']._serialized_end=2005 - _globals['_ARTIFACT']._serialized_start=2007 - _globals['_ARTIFACT']._serialized_end=2073 - _globals['_RELEASECONTENT']._serialized_start=2076 - _globals['_RELEASECONTENT']._serialized_end=2309 - _globals['_PATH']._serialized_start=2311 - _globals['_PATH']._serialized_end=2415 - _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=2418 - _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=2569 - _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=2572 - _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=2711 - _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=2714 - _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=2867 - _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=2870 - _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=3011 - _globals['_WORKFLOWSSERVICE']._serialized_start=3014 - _globals['_WORKFLOWSSERVICE']._serialized_end=4304 + _globals['_LISTWORKFLOWSREQUEST'].fields_by_name['page']._loaded_options = None + _globals['_LISTWORKFLOWSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' + _globals['_LISTWORKFLOWSRESPONSE'].fields_by_name['next_page']._loaded_options = None + _globals['_LISTWORKFLOWSRESPONSE'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' + _globals['_LISTPUBLICWORKFLOWSREQUEST'].fields_by_name['page']._loaded_options = None + _globals['_LISTPUBLICWORKFLOWSREQUEST'].fields_by_name['page']._serialized_options = b'\252\001\002\010\001' + _globals['_LISTPUBLICWORKFLOWSRESPONSE'].fields_by_name['next_page']._loaded_options = None + _globals['_LISTPUBLICWORKFLOWSRESPONSE'].fields_by_name['next_page']._serialized_options = b'\252\001\002\010\001' + _globals['_CLUSTER']._serialized_start=150 + _globals['_CLUSTER']._serialized_end=353 + _globals['_CREATECLUSTERREQUEST']._serialized_start=355 + _globals['_CREATECLUSTERREQUEST']._serialized_end=451 + _globals['_GETCLUSTERREQUEST']._serialized_start=453 + _globals['_GETCLUSTERREQUEST']._serialized_end=507 + _globals['_UPDATECLUSTERREQUEST']._serialized_start=509 + _globals['_UPDATECLUSTERREQUEST']._serialized_end=634 + _globals['_DELETECLUSTERREQUEST']._serialized_start=636 + _globals['_DELETECLUSTERREQUEST']._serialized_end=693 + _globals['_DELETECLUSTERRESPONSE']._serialized_start=695 + _globals['_DELETECLUSTERRESPONSE']._serialized_end=718 + _globals['_LISTCLUSTERSREQUEST']._serialized_start=720 + _globals['_LISTCLUSTERSREQUEST']._serialized_end=741 + _globals['_LISTCLUSTERSRESPONSE']._serialized_start=743 + _globals['_LISTCLUSTERSRESPONSE']._serialized_end=816 + _globals['_GETWORKFLOWREQUEST']._serialized_start=818 + _globals['_GETWORKFLOWREQUEST']._serialized_end=875 + _globals['_UPDATEWORKFLOWREQUEST']._serialized_start=878 + _globals['_UPDATEWORKFLOWREQUEST']._serialized_end=1006 + _globals['_DELETEWORKFLOWREQUEST']._serialized_start=1008 + _globals['_DELETEWORKFLOWREQUEST']._serialized_end=1068 + _globals['_DELETEWORKFLOWRESPONSE']._serialized_start=1070 + _globals['_DELETEWORKFLOWRESPONSE']._serialized_end=1094 + _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_start=1097 + _globals['_PUBLISHWORKFLOWRELEASEREQUEST']._serialized_end=1270 + _globals['_UNPUBLISHWORKFLOWRELEASEREQUEST']._serialized_start=1272 + _globals['_UNPUBLISHWORKFLOWRELEASEREQUEST']._serialized_end=1389 + _globals['_UNPUBLISHWORKFLOWRELEASERESPONSE']._serialized_start=1391 + _globals['_UNPUBLISHWORKFLOWRELEASERESPONSE']._serialized_end=1425 + _globals['_LISTWORKFLOWSREQUEST']._serialized_start=1427 + _globals['_LISTWORKFLOWSREQUEST']._serialized_end=1500 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_start=1503 + _globals['_LISTWORKFLOWSRESPONSE']._serialized_end=1640 + _globals['_LISTPUBLICWORKFLOWSREQUEST']._serialized_start=1642 + _globals['_LISTPUBLICWORKFLOWSREQUEST']._serialized_end=1721 + _globals['_LISTPUBLICWORKFLOWSRESPONSE']._serialized_start=1724 + _globals['_LISTPUBLICWORKFLOWSRESPONSE']._serialized_end=1867 + _globals['_CREATEWORKFLOWREQUEST']._serialized_start=1869 + _globals['_CREATEWORKFLOWREQUEST']._serialized_end=1946 + _globals['_WORKFLOW']._serialized_start=1949 + _globals['_WORKFLOW']._serialized_end=2098 + _globals['_WORKFLOWRELEASE']._serialized_start=2101 + _globals['_WORKFLOWRELEASE']._serialized_end=2368 + _globals['_ARTIFACT']._serialized_start=2370 + _globals['_ARTIFACT']._serialized_end=2436 + _globals['_RELEASECONTENT']._serialized_start=2439 + _globals['_RELEASECONTENT']._serialized_end=2672 + _globals['_PATH']._serialized_start=2674 + _globals['_PATH']._serialized_end=2778 + _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=2781 + _globals['_DEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=2932 + _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=2935 + _globals['_DEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=3074 + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_start=3077 + _globals['_UNDEPLOYWORKFLOWRELEASEREQUEST']._serialized_end=3230 + _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_start=3233 + _globals['_UNDEPLOYWORKFLOWRELEASERESPONSE']._serialized_end=3374 + _globals['_WORKFLOWSSERVICE']._serialized_start=3377 + _globals['_WORKFLOWSSERVICE']._serialized_end=4775 # @@protoc_insertion_point(module_scope) diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi index 6b6a48a..906b65f 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2.pyi @@ -1,5 +1,6 @@ from google.protobuf import timestamp_pb2 as _timestamp_pb2 from tilebox.datasets.tilebox.v1 import id_pb2 as _id_pb2 +from tilebox.datasets.tilebox.v1 import query_pb2 as _query_pb2 from tilebox.workflows.workflows.v1 import core_pb2 as _core_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor @@ -118,14 +119,32 @@ class UnpublishWorkflowReleaseResponse(_message.Message): def __init__(self) -> None: ... class ListWorkflowsRequest(_message.Message): - __slots__ = () - def __init__(self) -> None: ... + __slots__ = ("page",) + PAGE_FIELD_NUMBER: _ClassVar[int] + page: _query_pb2.Pagination + def __init__(self, page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... class ListWorkflowsResponse(_message.Message): - __slots__ = ("workflows",) + __slots__ = ("workflows", "next_page") + WORKFLOWS_FIELD_NUMBER: _ClassVar[int] + NEXT_PAGE_FIELD_NUMBER: _ClassVar[int] + workflows: _containers.RepeatedCompositeFieldContainer[Workflow] + next_page: _query_pb2.Pagination + def __init__(self, workflows: _Optional[_Iterable[_Union[Workflow, _Mapping]]] = ..., next_page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class ListPublicWorkflowsRequest(_message.Message): + __slots__ = ("page",) + PAGE_FIELD_NUMBER: _ClassVar[int] + page: _query_pb2.Pagination + def __init__(self, page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... + +class ListPublicWorkflowsResponse(_message.Message): + __slots__ = ("workflows", "next_page") WORKFLOWS_FIELD_NUMBER: _ClassVar[int] + NEXT_PAGE_FIELD_NUMBER: _ClassVar[int] workflows: _containers.RepeatedCompositeFieldContainer[Workflow] - def __init__(self, workflows: _Optional[_Iterable[_Union[Workflow, _Mapping]]] = ...) -> None: ... + next_page: _query_pb2.Pagination + def __init__(self, workflows: _Optional[_Iterable[_Union[Workflow, _Mapping]]] = ..., next_page: _Optional[_Union[_query_pb2.Pagination, _Mapping]] = ...) -> None: ... class CreateWorkflowRequest(_message.Message): __slots__ = ("name", "description") diff --git a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py index d6b0e13..0696724 100644 --- a/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py +++ b/tilebox-workflows/tilebox/workflows/workflows/v1/workflows_pb2_grpc.py @@ -50,6 +50,11 @@ def __init__(self, channel): request_serializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest.SerializeToString, response_deserializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse.FromString, _registered_method=True) + self.ListPublicWorkflows = channel.unary_unary( + '/workflows.v1.WorkflowsService/ListPublicWorkflows', + request_serializer=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest.SerializeToString, + response_deserializer=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse.FromString, + _registered_method=True) self.GetWorkflow = channel.unary_unary( '/workflows.v1.WorkflowsService/GetWorkflow', request_serializer=workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest.SerializeToString, @@ -133,6 +138,12 @@ def ListWorkflows(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ListPublicWorkflows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def GetWorkflow(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) @@ -213,6 +224,11 @@ def add_WorkflowsServiceServicer_to_server(servicer, server): request_deserializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsRequest.FromString, response_serializer=workflows_dot_v1_dot_workflows__pb2.ListWorkflowsResponse.SerializeToString, ), + 'ListPublicWorkflows': grpc.unary_unary_rpc_method_handler( + servicer.ListPublicWorkflows, + request_deserializer=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest.FromString, + response_serializer=workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse.SerializeToString, + ), 'GetWorkflow': grpc.unary_unary_rpc_method_handler( servicer.GetWorkflow, request_deserializer=workflows_dot_v1_dot_workflows__pb2.GetWorkflowRequest.FromString, @@ -449,6 +465,33 @@ def ListWorkflows(request, metadata, _registered_method=True) + @staticmethod + def ListPublicWorkflows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/workflows.v1.WorkflowsService/ListPublicWorkflows', + workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsRequest.SerializeToString, + workflows_dot_v1_dot_workflows__pb2.ListPublicWorkflowsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetWorkflow(request, target,