From 05526f4dd7c2e6a11b2437c548bd0e7b73b0c4f8 Mon Sep 17 00:00:00 2001
From: Maxim Podosochnyy <podosochnyy@perx.ru>
Date: Thu, 29 Jun 2023 15:08:37 +0700
Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?=
=?UTF-8?q?=D0=BD=20=D0=BF=D1=80=D0=B8=D0=BC=D0=B5=D1=80=20=D1=81=20=D1=81?=
=?UTF-8?q?=D0=B5=D1=80=D0=B2=D0=B8=D1=81=D0=BE=D0=BC=20=D1=80=D0=B0=D1=81?=
=?UTF-8?q?=D1=88=D0=B8=D1=80=D0=B5=D0=BD=D0=B8=D1=8F,=20=D0=B8=D1=81?=
=?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5=D1=82?=
=?UTF-8?q?=D0=BE=D0=B4=20update=5Fcollections=20=D0=B2=20extension=5Fsetu?=
=?UTF-8?q?p?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
README.md | 81 +------------
examples/extension_service/.gitignore | 1 +
examples/extension_service/Dockerfile | 19 +++
.../extension_service/constants/__init__.py | 0
.../constants/collections.py | 3 +
.../extension_service/constants/extension.py | 21 ++++
examples/extension_service/docker-compose.yml | 23 ++++
examples/extension_service/helpers.py | 114 ++++++++++++++++++
examples/extension_service/readme.md | 5 +
.../schemes/test_collection.json | 33 +++++
examples/extension_service/server.py | 62 ++++++++++
examples/extension_service/servicer.py | 39 ++++++
perxis/collections/helpers.py | 19 +++
perxis/extensions/extension_setup.py | 41 +++++--
perxis/interceptors.py | 69 +++++++++++
15 files changed, 442 insertions(+), 88 deletions(-)
create mode 100644 examples/extension_service/.gitignore
create mode 100644 examples/extension_service/Dockerfile
create mode 100644 examples/extension_service/constants/__init__.py
create mode 100644 examples/extension_service/constants/collections.py
create mode 100644 examples/extension_service/constants/extension.py
create mode 100644 examples/extension_service/docker-compose.yml
create mode 100644 examples/extension_service/helpers.py
create mode 100644 examples/extension_service/readme.md
create mode 100644 examples/extension_service/schemes/test_collection.json
create mode 100644 examples/extension_service/server.py
create mode 100644 examples/extension_service/servicer.py
create mode 100644 perxis/collections/helpers.py
create mode 100644 perxis/interceptors.py
diff --git a/README.md b/README.md
index 46d2701..cc96338 100644
--- a/README.md
+++ b/README.md
@@ -1,83 +1,14 @@
# perxis-python
## Расширения
-### Пример подключения расширения к Perxis
-...
+### Локальная разработка
+Для работы расширения требуется указание системного контекста РїСЂРё выполнении запросов. Рто возможно только РІ случае РїСЂСЏРјРѕР№
+работы с сервисами perxis минуя envoy, для этого они должны быть в одной сети. Поэтому нужен локально запущенный экземпляр
+perxis. Для его запуска нужно в каталоге perxis выполнить команду make run-local. После этого в контейнере в той же сети
+можно запустить сервис с расширением
### Пример написания сервиса расширений
-```python
-from perxis.extensions.extension_service import ExtensionService
-from perxis.roles import roles_pb2
-from perxis.common import common_pb2
-from perxis.clients import clients_pb2
-
-from perxis.collections import collections_pb2
-
-
-def make_collection_instances(schemes_mapping: dict[str, str]) -> list[collections_pb2.Collection]:
- collections = []
-
- for collection_id, collection_name in schemes_mapping.items():
- with open(f"./schemes/{collection_id}.json", "r") as file:
- collection_schema = file.read()
-
- collection = collections_pb2.Collection(
- id=collection_id,
- name=collection_name,
- schema=collection_schema
- )
-
- collections.append(collection)
-
- return collections
-
-...
-
-schemes_mapping = {
- "dealers_cities": "Дилеры/Города",
- "dealers_contacts": "Дилеры/Контакты",
- "dealers_countries": "Дилеры/Страны",
- "dealers_dealers": "Дилеры/Дилеры",
- "dealers_dealerships": "Дилеры/Дилерские центры",
- "dealers_department_types": "Дилеры/Типы отделов",
- "dealers_departments": "Дилеры/Отделы",
- "dealers_events": "Дилеры/События",
- "dealers_identifiers": "Дилеры/Рдентификаторы",
- "dealers_schedules": "Дилеры/Графики работы",
- "dealers_services": "Дилеры/Услуги"
-}
-
-...
-
-class Servicer(ExtensionService):
- extension_id = "some_id"
- collections = make_collection_instances(schemes_mapping)
- roles = [
- roles_pb2.Role(
- id="my-role",
- description="Описание к роли",
- rules=[
- common_pb2.Rule(
- collection_id="dealers_cities",
- actions=[common_pb2.CREATE, common_pb2.UPDATE, common_pb2.DELETE],
- )
- ],
- environments=["*"],
- allow_management=False,
- )
- ]
- clients = [
- clients_pb2.Client(
- id="my-client",
- name="Клиент, созданный расширением",
- description="Описание созданного расширением клиента",
- role_id="my-role",
- api_key={
- "rotate": True
- }
- )
- ]
-```
+Готовый пример с реализацией простого сервиса можно посмотреть в каталоге examples/extension_service
## Аутентификация
diff --git a/examples/extension_service/.gitignore b/examples/extension_service/.gitignore
new file mode 100644
index 0000000..fbc6772
--- /dev/null
+++ b/examples/extension_service/.gitignore
@@ -0,0 +1 @@
+.docker-compose.override.yml
\ No newline at end of file
diff --git a/examples/extension_service/Dockerfile b/examples/extension_service/Dockerfile
new file mode 100644
index 0000000..86eeb56
--- /dev/null
+++ b/examples/extension_service/Dockerfile
@@ -0,0 +1,19 @@
+FROM python:3.9-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV USER=perx
+
+RUN mkdir -p /home/${USER}/data /home/${USER}/app /home/${USER}/logs
+WORKDIR /home/${USER}/app
+
+ARG PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
+ENV PIP_EXTRA_INDEX_URL=$PIP_EXTRA_INDEX_URL
+
+COPY . /home/${USER}/app
+RUN pip install perxis==0.0.14
+
+ENV PYTHONPATH="/home/perx/app"
+ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+CMD ["python", "/home/perx/app/server.py"]
diff --git a/examples/extension_service/constants/__init__.py b/examples/extension_service/constants/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/examples/extension_service/constants/collections.py b/examples/extension_service/constants/collections.py
new file mode 100644
index 0000000..ef5a4a0
--- /dev/null
+++ b/examples/extension_service/constants/collections.py
@@ -0,0 +1,3 @@
+schemes_mapping = {
+ "test_collection": "Тестовая коллекция",
+}
diff --git a/examples/extension_service/constants/extension.py b/examples/extension_service/constants/extension.py
new file mode 100644
index 0000000..c623179
--- /dev/null
+++ b/examples/extension_service/constants/extension.py
@@ -0,0 +1,21 @@
+from perxis.extensions import manager_pb2
+
+
+ID = "demo-extension"
+NAME = "Демонстрационное расширение"
+VERSION = "v0.0.1"
+DESCRIPTION = "Демонстрационное расширение"
+VERSION_DESCRIPTION = "Описание расширения"
+DEPENDENCIES = []
+
+
+def get_extension_descriptor(host: str) -> manager_pb2.ExtensionDescriptor:
+ return manager_pb2.ExtensionDescriptor(
+ extension=ID,
+ title=NAME,
+ description=DESCRIPTION,
+ version=VERSION,
+ version_description=VERSION_DESCRIPTION,
+ deps=DEPENDENCIES,
+ url=host
+ )
diff --git a/examples/extension_service/docker-compose.yml b/examples/extension_service/docker-compose.yml
new file mode 100644
index 0000000..52b018e
--- /dev/null
+++ b/examples/extension_service/docker-compose.yml
@@ -0,0 +1,23 @@
+version: "3.9"
+
+services:
+ demo-ext-backend:
+ ports:
+ - 50051:50051
+ build:
+ context: .
+ args:
+ - PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
+ restart: unless-stopped
+ volumes:
+ - .:/home/perx/app
+ networks:
+ - storage
+ - default
+
+
+networks:
+ default:
+ storage:
+ external:
+ name: storage
diff --git a/examples/extension_service/helpers.py b/examples/extension_service/helpers.py
new file mode 100644
index 0000000..849c587
--- /dev/null
+++ b/examples/extension_service/helpers.py
@@ -0,0 +1,114 @@
+# TODO: убрать дублирование кода. На момент написания сервиса ещё не было версии пакета с perxis-python в которую входил
+# TODO: бы написанный ниже код. Удалить после того как станет не нужно
+
+import grpc
+import collections
+
+from typing import Optional
+
+from perxis.collections import collections_pb2
+from perxis.extensions import manager_pb2
+
+
+def make_descriptor(
+ extension_id: str, name: str, description: str,
+ version: str, version_description: str, host: str,
+ dependencies: Optional[list[str]] = None
+) -> manager_pb2.ExtensionDescriptor:
+ if dependencies is None:
+ dependencies = []
+
+ return manager_pb2.ExtensionDescriptor(
+ extension=extension_id,
+ title=name,
+ description=description,
+ version=version,
+ version_description=version_description,
+ deps=dependencies,
+ url=host
+ )
+
+
+def make_collection_instances(schemes_dir: str, schemes_mapping: dict[str, str]) -> list[collections_pb2.Collection]:
+ collections = []
+
+ for collection_id, collection_name in schemes_mapping.items():
+ with open(f"{schemes_dir}/{collection_id}.json", "r") as file:
+ collection_schema = file.read()
+
+ collection = collections_pb2.Collection(
+ id=collection_id,
+ name=collection_name,
+ schema=collection_schema
+ )
+
+ collections.append(collection)
+
+ return collections
+
+
+class _GenericClientInterceptor(grpc.UnaryUnaryClientInterceptor,
+ grpc.UnaryStreamClientInterceptor,
+ grpc.StreamUnaryClientInterceptor,
+ grpc.StreamStreamClientInterceptor):
+
+ def __init__(self, interceptor_function):
+ self._fn = interceptor_function
+
+ def intercept_unary_unary(self, continuation, client_call_details, request):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, iter((request,)), False, False)
+ response = continuation(new_details, next(new_request_iterator))
+ return postprocess(response) if postprocess else response
+
+ def intercept_unary_stream(self, continuation, client_call_details,
+ request):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, iter((request,)), False, True)
+ response_it = continuation(new_details, next(new_request_iterator))
+ return postprocess(response_it) if postprocess else response_it
+
+ def intercept_stream_unary(self, continuation, client_call_details,
+ request_iterator):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, request_iterator, True, False)
+ response = continuation(new_details, new_request_iterator)
+ return postprocess(response) if postprocess else response
+
+ def intercept_stream_stream(self, continuation, client_call_details,
+ request_iterator):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, request_iterator, True, True)
+ response_it = continuation(new_details, new_request_iterator)
+ return postprocess(response_it) if postprocess else response_it
+
+
+def create(intercept_call):
+ return _GenericClientInterceptor(intercept_call)
+
+
+class _ClientCallDetails(
+ collections.namedtuple(
+ '_ClientCallDetails',
+ ('method', 'timeout', 'metadata', 'credentials')),
+ grpc.ClientCallDetails):
+ pass
+
+
+def header_adder_interceptor(header, value):
+ def intercept_call(client_call_details, request_iterator, request_streaming,
+ response_streaming):
+ metadata = []
+ if client_call_details.metadata is not None:
+ metadata = list(client_call_details.metadata)
+ metadata.append((
+ header,
+ value,
+ ))
+
+ client_call_details = _ClientCallDetails(
+ client_call_details.method, client_call_details.timeout, metadata,
+ client_call_details.credentials)
+ return client_call_details, request_iterator, None
+
+ return create(intercept_call)
\ No newline at end of file
diff --git a/examples/extension_service/readme.md b/examples/extension_service/readme.md
new file mode 100644
index 0000000..fad54db
--- /dev/null
+++ b/examples/extension_service/readme.md
@@ -0,0 +1,5 @@
+Для запуска необходимо:
+1. Создать файл .env
+2. Указать переменную PIP_EXTRA_INDEX_URL
+3. docker-compose up
+
diff --git a/examples/extension_service/schemes/test_collection.json b/examples/extension_service/schemes/test_collection.json
new file mode 100644
index 0000000..3189935
--- /dev/null
+++ b/examples/extension_service/schemes/test_collection.json
@@ -0,0 +1,33 @@
+{
+ "ui": {
+ "options": {
+ "fields": [
+ "aaa",
+ "bbb"
+ ]
+ }
+ },
+ "type": "object",
+ "params": {
+ "inline": false,
+ "fields": {
+ "bbb": {
+ "title": "Ещё поле",
+ "ui": {
+ "widget": "StringInput"
+ },
+ "type": "string",
+ "params": {}
+ },
+ "aaa": {
+ "title": "Какое то поле",
+ "ui": {
+ "widget": "StringInput"
+ },
+ "type": "string",
+ "params": {}
+ }
+ }
+ },
+ "loaded": false
+}
diff --git a/examples/extension_service/server.py b/examples/extension_service/server.py
new file mode 100644
index 0000000..e3c45ff
--- /dev/null
+++ b/examples/extension_service/server.py
@@ -0,0 +1,62 @@
+import grpc
+
+from concurrent import futures
+from perxis.collections import collections_pb2_grpc
+from perxis.environments import environments_pb2_grpc
+from perxis.roles import roles_pb2_grpc
+from perxis.clients import clients_pb2_grpc
+from perxis.extensions import extension_pb2_grpc, manager_pb2_grpc, manager_pb2
+
+from servicer import Servicer
+from helpers import header_adder_interceptor
+from constants.extension import get_extension_descriptor
+
+
+def main():
+ my_extension_descriptor = get_extension_descriptor(host="demo-ext-backend:50051")
+
+ interceptor = header_adder_interceptor(
+ 'x-perxis-access', 'system'
+ )
+
+ with grpc.insecure_channel("extension-manager:9030") as extensions_manager_channel:
+ intercept_channel_extensions_manager_channel = grpc.intercept_channel(extensions_manager_channel, interceptor)
+
+ ext_manager_stub = manager_pb2_grpc.ExtensionManagerStub(intercept_channel_extensions_manager_channel)
+
+ registered_extensions: manager_pb2.ListExtensionsResponse = ext_manager_stub.ListExtensions(manager_pb2.ListExtensionsRequest())
+
+ for ext in registered_extensions.extensions:
+ if ext.extension == my_extension_descriptor.extension:
+ if ext.version != my_extension_descriptor.version:
+ ext_manager_stub.UnregisterExtensions(
+ manager_pb2.UnregisterExtensionsRequest(
+ extensions=[my_extension_descriptor]
+ )
+ )
+
+ ext_manager_stub.RegisterExtensions(manager_pb2.RegisterExtensionsRequest(
+ extensions=[my_extension_descriptor]
+ ))
+
+ with grpc.insecure_channel("content:9020") as content_channel:
+ intercepted_content_channel = grpc.intercept_channel(content_channel, interceptor)
+
+ collections_stub = collections_pb2_grpc.CollectionsStub(intercepted_content_channel)
+ roles_stub = roles_pb2_grpc.RolesStub(intercepted_content_channel)
+ clients_stub = clients_pb2_grpc.ClientsStub(intercepted_content_channel)
+ environments_stub = environments_pb2_grpc.EnvironmentsStub(intercepted_content_channel)
+
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+ extension_pb2_grpc.add_ExtensionServicer_to_server(
+ Servicer(
+ collections_stub, environments_stub, roles_stub, clients_stub
+ ), server
+ )
+ server.add_insecure_port("[::]:50051")
+ server.start()
+ server.wait_for_termination()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/extension_service/servicer.py b/examples/extension_service/servicer.py
new file mode 100644
index 0000000..82ec915
--- /dev/null
+++ b/examples/extension_service/servicer.py
@@ -0,0 +1,39 @@
+from constants import extension
+from helpers import make_collection_instances
+from constants import collections
+
+
+from perxis.extensions.extension_service import ExtensionService
+from perxis.roles import roles_pb2
+from perxis.common import common_pb2
+from perxis.clients import clients_pb2
+
+
+class Servicer(ExtensionService):
+ extension_id = extension.ID
+ collections = make_collection_instances("./schemes", collections.schemes_mapping)
+ roles = [
+ roles_pb2.Role(
+ id="demo-role",
+ description="Описание к роли",
+ rules=[
+ common_pb2.Rule(
+ collection_id="dealers_cities",
+ actions=[common_pb2.CREATE, common_pb2.UPDATE, common_pb2.DELETE],
+ )
+ ],
+ environments=["*"],
+ allow_management=False,
+ )
+ ]
+ clients = [
+ clients_pb2.Client(
+ id="demo-client",
+ name="Демонстрационный клиент",
+ description="Описание созданного расширением клиента",
+ role_id="demo-role",
+ api_key={
+ "rotate": True
+ }
+ )
+ ]
diff --git a/perxis/collections/helpers.py b/perxis/collections/helpers.py
new file mode 100644
index 0000000..06ad1e1
--- /dev/null
+++ b/perxis/collections/helpers.py
@@ -0,0 +1,19 @@
+from perxis.collections import collections_pb2
+
+
+def make_collection_instances(schemes_dir: str, schemes_mapping: dict[str, str]) -> list[collections_pb2.Collection]:
+ collections = []
+
+ for collection_id, collection_name in schemes_mapping.items():
+ with open(f"{schemes_dir}/{collection_id}.json", "r") as file:
+ collection_schema = file.read()
+
+ collection = collections_pb2.Collection(
+ id=collection_id,
+ name=collection_name,
+ schema=collection_schema
+ )
+
+ collections.append(collection)
+
+ return collections
\ No newline at end of file
diff --git a/perxis/extensions/extension_setup.py b/perxis/extensions/extension_setup.py
index fa1981b..2445a38 100644
--- a/perxis/extensions/extension_setup.py
+++ b/perxis/extensions/extension_setup.py
@@ -250,7 +250,7 @@ class ExtensionSetup:
def __update_collections(self, space_id: str, env_id: str) -> list[str]:
"""
- Метод __обновления__ коллекций подразумевает что сами коллекции уже созданы. Миграция окружения требуется
+ Метод __обновления__ коллекций. Миграция окружения требуется
только в случае если одна или несколько схем коллекций изменялись. Алгоритм работы:
1. Получить фактически существующую коллекцию из БД
2. Обновить её в perxis
@@ -271,22 +271,37 @@ class ExtensionSetup:
collection = get_response.collection
except grpc.RpcError as e:
- errors_list.append(f"Не удалось получить коллекцию {local_collection.id}, {e.details()}")
+ collection = None
- continue
+ cloned_collection = copy.deepcopy(local_collection)
+ cloned_collection.space_id = space_id
+ cloned_collection.env_id = env_id
- try:
- cloned_collection = copy.deepcopy(local_collection)
- cloned_collection.space_id = space_id
- cloned_collection.env_id = env_id
+ # Коллекция может быть не найдена в случае если она добавлена в новой версии
+ if not collection:
+ try:
+ create_response, state = self.collections_service.Create.with_call(
+ collections_pb2.CreateRequest(collection=cloned_collection)
+ )
- self.collections_service.Update.with_call(
- collections_pb2.UpdateRequest(collection=cloned_collection)
- )
- except grpc.RpcError as e:
- errors_list.append(f"Не удалось обновить коллекцию {local_collection.id}, {e.details()}")
+ collection = create_response.collection
+ except grpc.RpcError as e:
+ errors_list.append(f"Не удалось создать коллекцию {local_collection.id}, {e.details()}")
- continue
+ continue
+ else:
+ try:
+ cloned_collection = copy.deepcopy(local_collection)
+ cloned_collection.space_id = space_id
+ cloned_collection.env_id = env_id
+
+ self.collections_service.Update.with_call(
+ collections_pb2.UpdateRequest(collection=cloned_collection)
+ )
+ except grpc.RpcError as e:
+ errors_list.append(f"Не удалось обновить коллекцию {local_collection.id}, {e.details()}")
+
+ continue
diff = DeepDiff(
json.loads(collection.schema or "{}"),
diff --git a/perxis/interceptors.py b/perxis/interceptors.py
new file mode 100644
index 0000000..87eb273
--- /dev/null
+++ b/perxis/interceptors.py
@@ -0,0 +1,69 @@
+import grpc
+import collections
+
+
+class _GenericClientInterceptor(grpc.UnaryUnaryClientInterceptor,
+ grpc.UnaryStreamClientInterceptor,
+ grpc.StreamUnaryClientInterceptor,
+ grpc.StreamStreamClientInterceptor):
+
+ def __init__(self, interceptor_function):
+ self._fn = interceptor_function
+
+ def intercept_unary_unary(self, continuation, client_call_details, request):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, iter((request,)), False, False)
+ response = continuation(new_details, next(new_request_iterator))
+ return postprocess(response) if postprocess else response
+
+ def intercept_unary_stream(self, continuation, client_call_details,
+ request):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, iter((request,)), False, True)
+ response_it = continuation(new_details, next(new_request_iterator))
+ return postprocess(response_it) if postprocess else response_it
+
+ def intercept_stream_unary(self, continuation, client_call_details,
+ request_iterator):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, request_iterator, True, False)
+ response = continuation(new_details, new_request_iterator)
+ return postprocess(response) if postprocess else response
+
+ def intercept_stream_stream(self, continuation, client_call_details,
+ request_iterator):
+ new_details, new_request_iterator, postprocess = self._fn(
+ client_call_details, request_iterator, True, True)
+ response_it = continuation(new_details, new_request_iterator)
+ return postprocess(response_it) if postprocess else response_it
+
+
+def create(intercept_call):
+ return _GenericClientInterceptor(intercept_call)
+
+
+class _ClientCallDetails(
+ collections.namedtuple(
+ '_ClientCallDetails',
+ ('method', 'timeout', 'metadata', 'credentials')),
+ grpc.ClientCallDetails):
+ pass
+
+
+def header_adder_interceptor(header, value):
+ def intercept_call(client_call_details, request_iterator, request_streaming,
+ response_streaming):
+ metadata = []
+ if client_call_details.metadata is not None:
+ metadata = list(client_call_details.metadata)
+ metadata.append((
+ header,
+ value,
+ ))
+
+ client_call_details = _ClientCallDetails(
+ client_call_details.method, client_call_details.timeout, metadata,
+ client_call_details.credentials)
+ return client_call_details, request_iterator, None
+
+ return create(intercept_call)
--
GitLab