From ad6435e8161e965af7b6cb686115a0853b0688b6 Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Tue, 18 Mar 2025 14:45:50 +0300 Subject: [PATCH 1/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C=20=D1=83=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20item'=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/extension_service/Dockerfile.local | 2 +- examples/extension_service/servicer.py | 35 ++ perxis/extensions/actions.py | 2 - perxis/extensions/bootstrap.py | 1 + perxis/extensions/extension_service.py | 44 ++- perxis/extensions/extension_setup.py | 349 +++++++++++++++++++- perxis/extensions/utils.py | 2 +- perxis/items/models.py | 132 ++++++++ perxis/items/rules.py | 111 +++++++ perxis/items/utils.py | 30 ++ 10 files changed, 687 insertions(+), 21 deletions(-) create mode 100644 perxis/items/models.py create mode 100644 perxis/items/rules.py create mode 100644 perxis/items/utils.py diff --git a/examples/extension_service/Dockerfile.local b/examples/extension_service/Dockerfile.local index 1b2d306..ae28061 100644 --- a/examples/extension_service/Dockerfile.local +++ b/examples/extension_service/Dockerfile.local @@ -11,7 +11,7 @@ ARG PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV PIP_EXTRA_INDEX_URL=$PIP_EXTRA_INDEX_URL COPY . /home/${USER}/app -RUN pip install perxis==1.3.0 +RUN pip install perxis==1.8.2 RUN pip install 'watchdog[watchmedo]' ENV PYTHONPATH="/home/perx/app" diff --git a/examples/extension_service/servicer.py b/examples/extension_service/servicer.py index f83fd86..bcfca7c 100644 --- a/examples/extension_service/servicer.py +++ b/examples/extension_service/servicer.py @@ -4,6 +4,8 @@ from constants import extension from constants import collections as extension_collections +from perxis.items.utils import datasource_items_from_collections, sync_policies_from_collections +from perxis.items.models import DataSourceItem, SyncPolicyItem, Item, IfCollectionExists, IfExtensionInstalled from perxis.extensions.actions import make_action_dict from perxis.extensions import extension_service_pb2 from perxis.extensions.extension_service import ExtensionService @@ -76,6 +78,39 @@ class Servicer(ExtensionService): ) ] + # items = (datasource_items_from_collections(extension_collections.schemes_mapping) + # + sync_policies_from_collections(extension_collections.schemes_mapping)) + + items = [ + DataSourceItem( + collection_id="test_collection" + ), + SyncPolicyItem( + collection_id="test_collection", + ), + Item( + collection_id="secrets", + data={ + "id": f"{extension_id}__key", + "name": f"Ключ для {extension_id}", + "key": "123" + }, + rules=[IfExtensionInstalled("secrets")] + ), + Item( + collection_id="web_domains", + identifier_field="domain", + data={ + "domain": "example.com", + "primary": True, + "tls_ssl": "letsencrypt", + }, + rules=[ + IfCollectionExists() + ] + ) + ] + async def action_get_current_organization_and_users( self, request: extension_pb2.ActionRequest, diff --git a/perxis/extensions/actions.py b/perxis/extensions/actions.py index e2ae018..a84976a 100644 --- a/perxis/extensions/actions.py +++ b/perxis/extensions/actions.py @@ -1,7 +1,5 @@ import copy -from typing import Optional - from google.protobuf.struct_pb2 import Struct from perxis.items import items_pb2 diff --git a/perxis/extensions/bootstrap.py b/perxis/extensions/bootstrap.py index c3e5c3e..4e3eb66 100644 --- a/perxis/extensions/bootstrap.py +++ b/perxis/extensions/bootstrap.py @@ -100,6 +100,7 @@ async def _main( server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=10)) servicer = servicer_cls( + ext_manager_service=ext_manager_stub, collections_service=collections_stub, environments_service=environments_stub, roles_service=roles_stub, diff --git a/perxis/extensions/extension_service.py b/perxis/extensions/extension_service.py index 94a479e..dc98f82 100644 --- a/perxis/extensions/extension_service.py +++ b/perxis/extensions/extension_service.py @@ -9,7 +9,7 @@ import dataclasses from google.protobuf import timestamp_pb2, any_pb2, wrappers_pb2 -from perxis.extensions import extension_service_pb2, extension_service_pb2_grpc, extension_pb2 +from perxis.extensions import extension_service_pb2, extension_service_pb2_grpc, extension_pb2, manager_service_pb2_grpc from perxis.collaborators import collaborators_pb2_grpc from perxis.invitations import invitations_pb2_grpc from perxis.locales import locales_pb2_grpc @@ -27,6 +27,7 @@ from perxis.common import operation_pb2, operation_service_pb2_grpc, operation_s from perxis.collections import collections_pb2_grpc, collections_pb2 from perxis.environments import environments_pb2_grpc from perxis.extensions.extension_setup import ExtensionSetup +from perxis.items.models import AbstractItem def generate_operation_id() -> str: @@ -93,10 +94,12 @@ class ExtensionService( roles: list[roles_pb2.Role] = [] clients: list[clients_pb2.Client] = [] actions: list[dict] = [] + items: list[AbstractItem] = [] __operations: dict[str, OperationMeta] def __init__(self, + ext_manager_service: manager_service_pb2_grpc.ExtensionManagerServiceStub, collections_service: collections_pb2_grpc.CollectionsStub, environments_service: environments_pb2_grpc.EnvironmentsStub, roles_service: roles_pb2_grpc.RolesStub, @@ -115,6 +118,7 @@ class ExtensionService( channel: grpc.Channel, ): self.logger = logging.getLogger(__name__) + self.ext_manager_service = ext_manager_service self.collections_service = collections_service self.environments_service = environments_service self.roles_service = roles_service @@ -151,6 +155,20 @@ class ExtensionService( for action in self.actions or []: self.extension_setup.add_action(action) + services_list = [ + (service_name, getattr(self, service_name)) + for service_name + in self.__dict__ + if service_name.endswith("_service") + ] + services_list.append(("channel", self.channel, )) + + for item in self.items: + for rule in item.rules: + rule.bind_services(services_list) + + self.extension_setup.set_items(self.items) + @aiocron.crontab('0 * * * *', start=True) async def remove_old_operations(): self.remove_old_operations() @@ -251,10 +269,19 @@ class ExtensionService( request.space_id, request.env_id, request.force ) + errors_list += await self.additional_install_operations(operation_id, request, context) + self.result_log("установки", operation_id, request, errors_list) self.mark_operation_as_finished(operation_id, errors_list) + async def additional_install_operations(self, operation_id: str, request: extension_service_pb2.InstallRequest, context) -> list[str]: + """ + Для доп. логики установки + """ + + return [] + async def Install(self, request: extension_service_pb2.InstallRequest, context): operation_id = generate_operation_id() operation_description = "Установка расширения %s для окружения %s пространства %s. %s force" % ( @@ -274,11 +301,19 @@ class ExtensionService( async def _Uninstall(self, operation_id: str, request: extension_service_pb2.UninstallRequest, context): errors_list: list[str] = await self.extension_setup.uninstall(request.space_id, request.env_id, request.remove) + errors_list += await self.additional_uninstall_operations(operation_id, request, context) self.result_log("удаления", operation_id, request, errors_list) self.mark_operation_as_finished(operation_id, errors_list) + async def additional_uninstall_operations(self, operation_id: str, request: extension_service_pb2.UninstallRequest, context) -> list[str]: + """ + Для доп. логики удаления + """ + + return [] + async def Uninstall(self, request: extension_service_pb2.UninstallRequest, context): operation_id = generate_operation_id() operation_description = "Удаление расширения %s для окружения %s пространства %s. %s remove" % ( @@ -303,6 +338,13 @@ class ExtensionService( self.__operations[operation_id].mark_finished(errors_list) + async def additional_check_operations(self, operation_id: str, request: extension_service_pb2.CheckRequest, context) -> list[str]: + """" + Для доп. логики проверки + """ + + return [] + async def Check(self, request: extension_service_pb2.CheckRequest, context): operation_id = generate_operation_id() operation_description = "Проверка расширения %s для окружения %s пространства %s" % ( diff --git a/perxis/extensions/extension_setup.py b/perxis/extensions/extension_setup.py index 4fa4431..c613f56 100644 --- a/perxis/extensions/extension_setup.py +++ b/perxis/extensions/extension_setup.py @@ -16,6 +16,8 @@ from perxis.common import common_pb2 from perxis.clients import clients_pb2_grpc, clients_pb2 from perxis.environments import environments_pb2_grpc, environments_pb2 from perxis.extensions.actions import make_action_item, ACTIONS_COLLECTION_ID +from perxis.items.models import AbstractItem, SyncPolicyItem +from perxis.provider import PerxisItemsWrapper logger = logging.getLogger(__name__) @@ -34,6 +36,7 @@ class ExtensionSetup: self.clients = [] self.roles = [] self.actions = [] + self.items = [] self.roles_service = roles_service self.clients_service = clients_service @@ -68,6 +71,9 @@ class ExtensionSetup: def set_clients(self, clients: list[clients_pb2.Client]): self.clients = clients + def set_items(self, items: list[AbstractItem]): + self.items = items + # Работа с ролями async def __remove_roles(self, space_id: str) -> list[str]: errors_list: list[str] = [] @@ -571,40 +577,351 @@ class ExtensionSetup: return errors_list + async def __check_items(self, space_id: str, env_id: str) -> list[str]: + errors_list: list[str] = [] + wrapper = PerxisItemsWrapper( + self.items_service, + ) + + for item in self.items: + try: + if item.rules: + all_rules_satisfied = all( + [ + await rule(item, space_id, env_id) + for rule + in item.rules + ] + ) + + if not all_rules_satisfied: + continue + + message = await wrapper.find( + collection_id=item.collection_id, + env_id=env_id, + space_id=space_id, + limit=1, + offset=0, + fields=["id"], + filters=[f"{item.identifier_field} == '{item.identifier}'"] + ) + + if not message.total: + errors_list.append(f"Item {item.identifier} не найден") + except Exception as e: + errors_list.append( + f"Не удалось проверить item {item.identifier} " + f"коллекции {item.collection_id} - {e}" + ) + + return errors_list + + async def __update_items(self, space_id: str, env_id: str) -> list[str]: + errors_list: list[str] = [] + wrapper = PerxisItemsWrapper( + self.items_service, + ) + + for item in self.items: + if item.rules: + all_rules_satisfied = all( + [ + await rule(item, space_id, env_id) + for rule + in item.rules + ] + ) + + if not all_rules_satisfied: + continue + + try: + message = await wrapper.find( + collection_id=item.collection_id, + env_id=env_id, + space_id=space_id, + limit=1, + offset=0, + filters=[f"{item.identifier_field} == '{item.identifier}'"] + ) + + if message.items: + item_in_perxis = message.items[0] + else: + item_in_perxis = None + except Exception as e: + if hasattr(e, "details") and "not found" in e.details(): + is_error = False + else: + is_error = True + + if is_error: + errors_list.append( + f"Не удалось получить item {item.identifier} " + f"коллекции {item.collection_id} - {e}" + ) + + continue + + try: + if item_in_perxis: + # Для того чтобы не затереть изменения в perxis + # Нужно смержить данные. Логика работы: + # 1. Данные которые указаны в `data` в расширении - в приоритете, они замещают то что в perxis + # 2. Данные которые есть в perxis но нет в расширении - дополняются + + await wrapper.update( + collection_id=item.collection_id, + item_id=item_in_perxis.id, + space_id=space_id, + env_id=env_id, + data=item.merge_data(MessageToDict(item_in_perxis)["data"]), + ) + else: + message = await wrapper.create( + collection_id=item.collection_id, + space_id=space_id, + env_id=env_id, + data=item.struct, + ) + + item_in_perxis = message.created + + await wrapper.publish( + item_id=item_in_perxis.id, + collection_id=item.collection_id, + space_id=space_id, + env_id=env_id + ) + except Exception as e: + errors_list.append( + f"Не удалось записать item {item.identifier} " + f"коллекции {item.collection_id} - {e}" + ) + + continue + + return errors_list + + async def __remove_items(self, space_id: str, env_id: str) -> list[str]: + errors_list: list[str] = [] + wrapper = PerxisItemsWrapper( + self.items_service, + ) + + for item in self.items: + if item.rules: + all_rules_satisfied = all( + [ + await rule(item, space_id, env_id) + for rule + in item.rules + ] + ) + + if not all_rules_satisfied: + continue + + try: + message = await wrapper.find( + collection_id=item.collection_id, + env_id=env_id, + space_id=space_id, + limit=1, + offset=0, + fields=["id"], + filters=[f"{item.identifier_field} == '{item.identifier}'"] + ) + + if message.items: + await wrapper.delete( + item_id=message.items[0].id, + collection_id=message.items[0].collection_id, + space_id=space_id, + env_id=env_id, + ) + except Exception as e: + if hasattr(e, "details") and "not found" in e.details(): + is_error = False + else: + is_error = True + + if is_error: + errors_list.append( + f"Не удалось удалить item {item.identifier} " + f"коллекции {item.collection_id} - {e}" + ) + + return errors_list + + async def __update_view_role(self, space_id: str, env_id: str, mode: str = "add") -> list[str]: + errors = [] + + # Нужны только относящиеся к синхронизации элементы + items_for_view_role = [ + item + for item + in self.items + if (isinstance(item, SyncPolicyItem) or item.collection_id == "hoop_item_sync_policies") + and item.data["export_view"] + and await item.all_rules_is_satisfied(space_id=space_id, env_id=env_id) + ] + + if not items_for_view_role: + return errors + + try: + message = await self.roles_service.Get( + roles_pb2.GetRequest( + space_id=space_id, + role_id="view" + ) + ) + + role = message.role + except grpc.RpcError as e: + if "not found" not in e.details(): + errors.append(f"Не удалось получить роль view, {e.details()}") + + role = None + + if not role: + try: + message = await self.roles_service.Create( + roles_pb2.CreateRequest( + role=roles_pb2.Role( + id="view", + space_id=space_id, + description="Роль для view коллекций", + rules=[], + environments=["*"], + allow_management=False, + ) + ), + ) + + role = message.created + except grpc.RpcError as e: + errors.append(f"Не удалось создать роль view, {e.details()}") + + return errors + + # Произвести мерж правил доступа + for item in items_for_view_role: + actions_from_item = [] + + if not item.data["deny_read"]: + actions_from_item.append(common_pb2.READ) + + if not item.data["deny_create"]: + actions_from_item.append(common_pb2.CREATE) + + if not item.data["deny_publish"]: + actions_from_item.append(common_pb2.UPDATE) + + if not item.data["deny_delete"]: + actions_from_item.append(common_pb2.DELETE) + + rule_was_found = False + for rule in role.rules: + if rule.collection_id != item.data["collection"]: + continue + + rule_was_found = True + + if mode == "add": + modified_actions = list(set(list(rule.actions) + actions_from_item)) + else: + modified_actions = [ + action + for action + in rule.actions + if action not in actions_from_item + ] + + rule.actions[:] = modified_actions + + # Если правила для коллекции нет и при этом доступы нужно __добавлять__ + if not rule_was_found and mode == "add": + role.rules.append( + common_pb2.Rule( + collection_id=item.data["collection"], + actions=actions_from_item, + ) + ) + + try: + await self.roles_service.Update( + roles_pb2.UpdateRequest( + role=role + ) + ) + except grpc.RpcError as e: + errors.append(f"Не удалось обновить роль view, {e.details()}") + + return errors + + async def install(self, space_id: str, env_id: str, use_force: bool) -> list[str]: errors = [] - errors += await self.__update_collections(space_id, env_id) - errors += await self.__update_roles(space_id, env_id) - errors += await self.__update_clients(space_id) - errors += await self.__update_actions(space_id, env_id) + try: + errors += await self.__update_collections(space_id, env_id) + errors += await self.__update_roles(space_id, env_id) + errors += await self.__update_clients(space_id) + errors += await self.__update_actions(space_id, env_id) + errors += await self.__update_items(space_id, env_id) + errors += await self.__update_view_role(space_id, env_id) + except Exception as e: + logger.exception(e) + errors.append(f"Во время установки было необработанное исключение - {e}") return errors async def update(self, space_id: str, env_id: str, use_force: bool) -> list[str]: errors = [] - errors += await self.__update_collections(space_id, env_id) - errors += await self.__update_roles(space_id, env_id) - errors += await self.__update_clients(space_id) - errors += await self.__update_actions(space_id, env_id) + try: + errors += await self.__update_collections(space_id, env_id) + errors += await self.__update_roles(space_id, env_id) + errors += await self.__update_clients(space_id) + errors += await self.__update_actions(space_id, env_id) + errors += await self.__update_items(space_id, env_id) + errors += await self.__update_view_role(space_id, env_id) + except Exception as e: + logger.exception(e) + errors.append(f"Во время обновления было необработанное исключение - {e}") return errors async def check(self, space_id: str, env_id: str) -> list[str]: - errors = await self.__check_collections(space_id, env_id) - errors += await self.__check_roles(space_id) - errors += await self.__check_clients(space_id) - errors += await self.__check_actions(space_id, env_id) + errors = [] + + try: + errors += await self.__check_collections(space_id, env_id) + errors += await self.__check_roles(space_id) + errors += await self.__check_clients(space_id) + errors += await self.__check_actions(space_id, env_id) + errors += await self.__check_items(space_id, env_id) + except Exception as e: + logger.exception(e) + errors.append(f"Во время проверки было необработанное исключение - {e}") return errors async def uninstall(self, space_id: str, env_id: str, use_remove: bool) -> list[str]: errors = [] if use_remove: - errors += await self.__remove_collections(space_id, env_id) - errors += await self.__remove_clients(space_id) - errors += await self.__remove_roles(space_id) - errors += await self.__remove_actions(space_id, env_id) + try: + errors += await self.__remove_collections(space_id, env_id) + errors += await self.__remove_clients(space_id) + errors += await self.__remove_roles(space_id) + errors += await self.__remove_actions(space_id, env_id) + errors += await self.__remove_items(space_id, env_id) + errors += await self.__update_view_role(space_id, env_id, "remove") + except Exception as e: + logger.exception(e) + errors.append(f"Во время удаления данных было необработанное исключение - {e}") return errors diff --git a/perxis/extensions/utils.py b/perxis/extensions/utils.py index b5bbc41..1075b3b 100644 --- a/perxis/extensions/utils.py +++ b/perxis/extensions/utils.py @@ -19,4 +19,4 @@ def get_extension_descriptor( version_description=ext_version_description, deps=ext_deps, url=ext_host - ) + ) \ No newline at end of file diff --git a/perxis/items/models.py b/perxis/items/models.py new file mode 100644 index 0000000..0b13d02 --- /dev/null +++ b/perxis/items/models.py @@ -0,0 +1,132 @@ +import abc + +from google.protobuf.struct_pb2 import Struct + +from perxis.items.rules import AbstractRule, IfCollectionExists, IfExtensionInstalled + + +class AbstractItem(metaclass=abc.ABCMeta): + """ + Абстрактный класс для item'а. Нужен для определения общих свойств без реализации какого + то конкретного конструктора. + """ + + collection_id: str + data: dict + + rules: list[AbstractRule] + identifier_field: str = "id" + + @property + def identifier(self): + return self.data[self.identifier_field] + + @property + def struct(self) -> Struct: + s = Struct() + s.update(self.data) + + return s + + async def all_rules_is_satisfied(self, space_id: str, env_id: str) -> bool: + return all( + [ + await rule(item=self, space_id=space_id, env_id=env_id) + for rule + in self.rules + ] + ) + + def merge_data(self, data_from_perxis: dict) -> Struct: + """ + Мерж данных из перксиса и расширения. В приоритете данные расширения, если что-то + из них было вручную изменено в perxis - значение будет затёрто + """ + + s = Struct() + s.update({ + **data_from_perxis, + **self.data, + }) + + return s + + +class Item(AbstractItem): + """ + Общий класс для любого Item + """ + + def __init__( + self, + collection_id: str, + data: dict, + identifier_field: str = "id", + rules: list[AbstractRule] | None = None + ): + self.collection_id = collection_id + self.data = data + self.identifier_field = identifier_field + self.rules = rules or [] + + +class DataSourceItem(AbstractItem): + """ + Класс для системной коллекции web_datasources + """ + + def __init__( + self, + collection_id: str, + rules: list[AbstractRule] | None = None, + query: str = "", + content_type: str = "", + exclude: bool = False + ): + self.collection_id = "web_datasources" + self.rules = rules or [IfCollectionExists()] + self.data = { + "id": collection_id, + "collection": collection_id, + "query": query, + "content_type": content_type, + "exclude": exclude, + } + + +class SyncPolicyItem(AbstractItem): + """ + Класс для коллекции hoop_item_sync_policies расширения perxishoop + """ + + identifier_field = "collection" + + def __init__( + self, + collection_id: str, + name: str = "", + key: str = "id", + export_view: bool = True, + remove_collection: bool = False, + deny_publish: bool = True, + deny_delete: bool = True, + deny_create: bool = True, + deny_read: bool = False, + hidden: bool = False, + rules: list[AbstractRule] | None = None, + ): + self.collection_id = "hoop_item_sync_policies" + self.rules = rules or [IfExtensionInstalled("perxishoop")] + + self.data = { + "name": name or f"Коллекция {collection_id}", + "collection": collection_id, + "key": key, + "export_view": export_view, + "remove_collection": remove_collection, + "deny_publish": deny_publish, + "deny_delete": deny_delete, + "deny_create": deny_create, + "deny_read": deny_read, + "hidden": hidden, + } \ No newline at end of file diff --git a/perxis/items/rules.py b/perxis/items/rules.py new file mode 100644 index 0000000..fcd2a6d --- /dev/null +++ b/perxis/items/rules.py @@ -0,0 +1,111 @@ +import abc +import typing +import logging + +from perxis.collections import collections_pb2 +from perxis.extensions import manager_service_pb2 + + +if typing.TYPE_CHECKING: + from perxis.items.models import AbstractItem + + +class AbstractRule(metaclass=abc.ABCMeta): + """ + Абстрактное правило для обработки создания item'ов в системе. Правила могут + использоваться в кейсах: + 1. В случае если item опционален и невозможность его создания не приведёт к невозможности + использовать само расширение + 2. Если при создании item'а нужно обогатить его данные чем-то внешним, не относящимся к + текущему расширению. Например - при создании item'ов для цепочки сервиса уведомлений. + Каждый последующий item может с помощью правил получать результат создания предыдущего + и на базе этих данных формировать ref + + Так как сервисов в ExtensionService очень много, для того чтобы код их проброса в объект rule + не был излишне длинным - это делается неявным пробросом свойств из ExtensionService. Таким + образом и код короче получается, и в случае если будут добавлены какие то новые сервисы они + автоматически будут проброшены сюда. + """ + + @property + def rule_name(self): + return self.__class__.__name__ + + @property + def logger(self): + return logging.getLogger(self.rule_name) + + def bind_services(self, service_list: list[tuple[str, object]]): + """ + Нужно для автоматического подключения сервисов из ExtensionService. Проблема + в том что их очень много и если руками явно указывать - будет большой boilerplate + """ + + for service_name, service in service_list: + setattr(self, service_name, service) + + @abc.abstractmethod + async def act(self, item: "AbstractItem", space_id: str, env_id: str): + raise NotImplementedError() + + async def __call__(self, item: "AbstractItem", space_id: str, env_id: str) -> bool: + try: + await self.act(item, space_id, env_id) + + return True + except Exception as e: + self.logger.warning( + f"Правило {self.rule_name} не " + f"сработало для {item.collection_id} (item {item.identifier}), " + f"ошибка - {e}" + ) + + return False + + +class IfCollectionExists(AbstractRule): + """ + Правило для опциональных item'ов которые нужно создавать в случае наличия + определённой коллекции в perxis. Может быть полезно если коллекция широко используется + но в данный момент устанавливается только вручную и не привязана ни к одному из расширений + """ + + async def act(self, item: "AbstractItem", space_id: str, env_id: str): + message = await self.collections_service.Get( + collections_pb2.GetRequest( + space_id=space_id, + env_id=env_id, + collection_id=item.collection_id + ) + ) + + if not message.collection: + raise ValueError(f"Коллекция {item.collection_id} не найдена") + + +class IfExtensionInstalled(AbstractRule): + """ + Правило для опциональных item'ов которые нужно создавать только в случае + наличия определённого расширения в perxis + """ + + required_extension: str + + def __init__(self, required_extension: str): + self.required_extension = required_extension + + async def act(self, item: "AbstractItem", space_id: str, env_id: str): + registered_extensions = await self.ext_manager_service.ListRegisteredExtensions( + manager_service_pb2.ListRegisteredExtensionsRequest() + ) + + ext_is_installed = False + for ext in registered_extensions.extensions: + if ext.extension == self.required_extension: + ext_is_installed = True + break + + if not ext_is_installed: + raise ValueError( + f"Расширение {self.required_extension} не установлено в perxis" + ) diff --git a/perxis/items/utils.py b/perxis/items/utils.py new file mode 100644 index 0000000..dd91aa4 --- /dev/null +++ b/perxis/items/utils.py @@ -0,0 +1,30 @@ +from perxis.items.models import DataSourceItem, SyncPolicyItem + + +def datasource_items_from_collections(collections_map: dict[str, str]) -> list[DataSourceItem]: + """ + Создание записей источников данных на базе маппинга коллекций + """ + + return [ + DataSourceItem( + collection_id=collection_id + ) + for collection_id in collections_map + if collection_id + ] + + +def sync_policies_from_collections(collections_map: dict[str, str]) -> list[SyncPolicyItem]: + """ + Создание записей синхронизации коллекций на базе маппинга коллекций + """ + + return [ + SyncPolicyItem( + collection_id=collection_id, + name=collection_name, + ) + for collection_id, collection_name in collections_map.items() + if collection_id + ] -- GitLab From 212d467c539b52a640f9516a0554ba3bb54ef35d Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Tue, 18 Mar 2025 14:58:49 +0300 Subject: [PATCH 2/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C=20=D1=83=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20item'=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perxis/items/models.py | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/perxis/items/models.py b/perxis/items/models.py index 0b13d02..bbbfeca 100644 --- a/perxis/items/models.py +++ b/perxis/items/models.py @@ -29,6 +29,9 @@ class AbstractItem(metaclass=abc.ABCMeta): return s async def all_rules_is_satisfied(self, space_id: str, env_id: str) -> bool: + if not self.rules: + return True + return all( [ await rule(item=self, space_id=space_id, env_id=env_id) diff --git a/setup.py b/setup.py index 73941b0..78b8248 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def load_requirements(): setup( name='perxis', - version='1.8.2', + version='1.9.0', description='Perxis python client', long_description=long_description, long_description_content_type='text/markdown', -- GitLab From 501d8125e12669614323aef9e1b9501def21480d Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Wed, 19 Mar 2025 10:25:27 +0300 Subject: [PATCH 3/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C=20=D1=83=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20item'=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/extension_service/Dockerfile | 2 +- examples/extension_service/Dockerfile.local | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/extension_service/Dockerfile b/examples/extension_service/Dockerfile index 8e11f94..3335c7a 100644 --- a/examples/extension_service/Dockerfile +++ b/examples/extension_service/Dockerfile @@ -11,7 +11,7 @@ ARG PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV PIP_EXTRA_INDEX_URL=$PIP_EXTRA_INDEX_URL COPY . /home/${USER}/app -RUN pip install perxis==1.3.0 +RUN pip install perxis==1.10.0 ENV PYTHONPATH="/home/perx/app" ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python diff --git a/examples/extension_service/Dockerfile.local b/examples/extension_service/Dockerfile.local index ae28061..9118f20 100644 --- a/examples/extension_service/Dockerfile.local +++ b/examples/extension_service/Dockerfile.local @@ -11,7 +11,7 @@ ARG PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} ENV PIP_EXTRA_INDEX_URL=$PIP_EXTRA_INDEX_URL COPY . /home/${USER}/app -RUN pip install perxis==1.8.2 +RUN pip install perxis==1.10.0 RUN pip install 'watchdog[watchmedo]' ENV PYTHONPATH="/home/perx/app" diff --git a/setup.py b/setup.py index 78b8248..2fb87ab 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def load_requirements(): setup( name='perxis', - version='1.9.0', + version='1.10.0', description='Perxis python client', long_description=long_description, long_description_content_type='text/markdown', -- GitLab From 87c65b211f5c25887059d0ca181aad0054c6322b Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Mon, 24 Mar 2025 15:47:08 +0300 Subject: [PATCH 4/7] =?UTF-8?q?=D0=9F=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/extension_service/servicer.py | 4 +-- perxis/extensions/extension_service.py | 2 +- perxis/extensions/extension_setup.py | 2 +- .../models.py => extensions/item_models.py} | 2 +- .../rules.py => extensions/item_rules.py} | 2 +- perxis/extensions/utils.py | 32 ++++++++++++++++++- perxis/items/utils.py | 30 ----------------- 7 files changed, 37 insertions(+), 37 deletions(-) rename perxis/{items/models.py => extensions/item_models.py} (97%) rename perxis/{items/rules.py => extensions/item_rules.py} (98%) delete mode 100644 perxis/items/utils.py diff --git a/examples/extension_service/servicer.py b/examples/extension_service/servicer.py index bcfca7c..f40984e 100644 --- a/examples/extension_service/servicer.py +++ b/examples/extension_service/servicer.py @@ -4,8 +4,8 @@ from constants import extension from constants import collections as extension_collections -from perxis.items.utils import datasource_items_from_collections, sync_policies_from_collections -from perxis.items.models import DataSourceItem, SyncPolicyItem, Item, IfCollectionExists, IfExtensionInstalled +from perxis.extensions.utils import datasource_items_from_collections, sync_policies_from_collections +from perxis.extensions.item_models import DataSourceItem, SyncPolicyItem, Item, IfCollectionExists, IfExtensionInstalled from perxis.extensions.actions import make_action_dict from perxis.extensions import extension_service_pb2 from perxis.extensions.extension_service import ExtensionService diff --git a/perxis/extensions/extension_service.py b/perxis/extensions/extension_service.py index dc98f82..3799b34 100644 --- a/perxis/extensions/extension_service.py +++ b/perxis/extensions/extension_service.py @@ -27,7 +27,7 @@ from perxis.common import operation_pb2, operation_service_pb2_grpc, operation_s from perxis.collections import collections_pb2_grpc, collections_pb2 from perxis.environments import environments_pb2_grpc from perxis.extensions.extension_setup import ExtensionSetup -from perxis.items.models import AbstractItem +from perxis.extensions.item_models import AbstractItem def generate_operation_id() -> str: diff --git a/perxis/extensions/extension_setup.py b/perxis/extensions/extension_setup.py index c613f56..41e9f55 100644 --- a/perxis/extensions/extension_setup.py +++ b/perxis/extensions/extension_setup.py @@ -16,7 +16,7 @@ from perxis.common import common_pb2 from perxis.clients import clients_pb2_grpc, clients_pb2 from perxis.environments import environments_pb2_grpc, environments_pb2 from perxis.extensions.actions import make_action_item, ACTIONS_COLLECTION_ID -from perxis.items.models import AbstractItem, SyncPolicyItem +from perxis.extensions.item_models import AbstractItem, SyncPolicyItem from perxis.provider import PerxisItemsWrapper diff --git a/perxis/items/models.py b/perxis/extensions/item_models.py similarity index 97% rename from perxis/items/models.py rename to perxis/extensions/item_models.py index bbbfeca..036f3ba 100644 --- a/perxis/items/models.py +++ b/perxis/extensions/item_models.py @@ -2,7 +2,7 @@ import abc from google.protobuf.struct_pb2 import Struct -from perxis.items.rules import AbstractRule, IfCollectionExists, IfExtensionInstalled +from perxis.extensions.item_rules import AbstractRule, IfCollectionExists, IfExtensionInstalled class AbstractItem(metaclass=abc.ABCMeta): diff --git a/perxis/items/rules.py b/perxis/extensions/item_rules.py similarity index 98% rename from perxis/items/rules.py rename to perxis/extensions/item_rules.py index fcd2a6d..106a98e 100644 --- a/perxis/items/rules.py +++ b/perxis/extensions/item_rules.py @@ -7,7 +7,7 @@ from perxis.extensions import manager_service_pb2 if typing.TYPE_CHECKING: - from perxis.items.models import AbstractItem + from perxis.extensions.item_models import AbstractItem class AbstractRule(metaclass=abc.ABCMeta): diff --git a/perxis/extensions/utils.py b/perxis/extensions/utils.py index 1075b3b..b5ba8e6 100644 --- a/perxis/extensions/utils.py +++ b/perxis/extensions/utils.py @@ -1,6 +1,36 @@ from typing import Optional from perxis.extensions import manager_service_pb2 +from perxis.extensions.item_models import DataSourceItem, SyncPolicyItem + + +def datasource_items_from_collections(collections_map: dict[str, str]) -> list[DataSourceItem]: + """ + Создание записей источников данных на базе маппинга коллекций + """ + + return [ + DataSourceItem( + collection_id=collection_id + ) + for collection_id in collections_map + if collection_id + ] + + +def sync_policies_from_collections(collections_map: dict[str, str]) -> list[SyncPolicyItem]: + """ + Создание записей синхронизации коллекций на базе маппинга коллекций + """ + + return [ + SyncPolicyItem( + collection_id=collection_id, + name=collection_name, + ) + for collection_id, collection_name in collections_map.items() + if collection_id + ] def get_extension_descriptor( @@ -19,4 +49,4 @@ def get_extension_descriptor( version_description=ext_version_description, deps=ext_deps, url=ext_host - ) \ No newline at end of file + ) diff --git a/perxis/items/utils.py b/perxis/items/utils.py deleted file mode 100644 index dd91aa4..0000000 --- a/perxis/items/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -from perxis.items.models import DataSourceItem, SyncPolicyItem - - -def datasource_items_from_collections(collections_map: dict[str, str]) -> list[DataSourceItem]: - """ - Создание записей источников данных на базе маппинга коллекций - """ - - return [ - DataSourceItem( - collection_id=collection_id - ) - for collection_id in collections_map - if collection_id - ] - - -def sync_policies_from_collections(collections_map: dict[str, str]) -> list[SyncPolicyItem]: - """ - Создание записей синхронизации коллекций на базе маппинга коллекций - """ - - return [ - SyncPolicyItem( - collection_id=collection_id, - name=collection_name, - ) - for collection_id, collection_name in collections_map.items() - if collection_id - ] -- GitLab From dae36cef8afd33a36701070bc7cdb55a955ab21e Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Mon, 24 Mar 2025 16:19:45 +0300 Subject: [PATCH 5/7] =?UTF-8?q?=D0=9F=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perxis/extensions/item_models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/perxis/extensions/item_models.py b/perxis/extensions/item_models.py index 036f3ba..f32e69f 100644 --- a/perxis/extensions/item_models.py +++ b/perxis/extensions/item_models.py @@ -29,9 +29,6 @@ class AbstractItem(metaclass=abc.ABCMeta): return s async def all_rules_is_satisfied(self, space_id: str, env_id: str) -> bool: - if not self.rules: - return True - return all( [ await rule(item=self, space_id=space_id, env_id=env_id) -- GitLab From db7c727f36d17ddda3490bd14383aab84e265dc3 Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Tue, 25 Mar 2025 14:20:17 +0300 Subject: [PATCH 6/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D1=84=D0=BB=D0=B0=D0=B3=D0=B8=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20=D1=83=D0=BA=D0=B0=D0=B7=D0=B0=D0=BD=D0=B8=D1=8F?= =?UTF-8?q?=20=D0=B4=D0=BE=D0=BF=D1=83=D1=81=D0=B8=D0=BC=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B8=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D1=8F?= =?UTF-8?q?=20/=20=D1=83=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20item'?= =?UTF-8?q?=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perxis/extensions/extension_setup.py | 9 ++++++++- perxis/extensions/item_models.py | 20 ++++++++++++++++++-- perxis/extensions/utils.py | 18 +++++++++++++++--- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/perxis/extensions/extension_setup.py b/perxis/extensions/extension_setup.py index 41e9f55..918f201 100644 --- a/perxis/extensions/extension_setup.py +++ b/perxis/extensions/extension_setup.py @@ -666,11 +666,14 @@ class ExtensionSetup: try: if item_in_perxis: + # Если установлен запрет на изменение item'ов + if not item.with_update: + continue + # Для того чтобы не затереть изменения в perxis # Нужно смержить данные. Логика работы: # 1. Данные которые указаны в `data` в расширении - в приоритете, они замещают то что в perxis # 2. Данные которые есть в perxis но нет в расширении - дополняются - await wrapper.update( collection_id=item.collection_id, item_id=item_in_perxis.id, @@ -723,6 +726,10 @@ class ExtensionSetup: if not all_rules_satisfied: continue + # Если установлен запрет на удаление item'ов + if not item.with_delete: + continue + try: message = await wrapper.find( collection_id=item.collection_id, diff --git a/perxis/extensions/item_models.py b/perxis/extensions/item_models.py index f32e69f..078ed22 100644 --- a/perxis/extensions/item_models.py +++ b/perxis/extensions/item_models.py @@ -16,6 +16,8 @@ class AbstractItem(metaclass=abc.ABCMeta): rules: list[AbstractRule] identifier_field: str = "id" + with_update: bool + with_delete: bool @property def identifier(self): @@ -61,6 +63,8 @@ class Item(AbstractItem): self, collection_id: str, data: dict, + with_update: bool = True, + with_delete: bool = True, identifier_field: str = "id", rules: list[AbstractRule] | None = None ): @@ -68,6 +72,8 @@ class Item(AbstractItem): self.data = data self.identifier_field = identifier_field self.rules = rules or [] + self.with_update = with_update + self.with_delete = with_delete class DataSourceItem(AbstractItem): @@ -81,7 +87,9 @@ class DataSourceItem(AbstractItem): rules: list[AbstractRule] | None = None, query: str = "", content_type: str = "", - exclude: bool = False + exclude: bool = False, + with_update: bool = True, + with_delete: bool = True, ): self.collection_id = "web_datasources" self.rules = rules or [IfCollectionExists()] @@ -93,6 +101,9 @@ class DataSourceItem(AbstractItem): "exclude": exclude, } + self.with_update = with_update + self.with_delete = with_delete + class SyncPolicyItem(AbstractItem): """ @@ -114,6 +125,8 @@ class SyncPolicyItem(AbstractItem): deny_read: bool = False, hidden: bool = False, rules: list[AbstractRule] | None = None, + with_update: bool = True, + with_delete: bool = True, ): self.collection_id = "hoop_item_sync_policies" self.rules = rules or [IfExtensionInstalled("perxishoop")] @@ -129,4 +142,7 @@ class SyncPolicyItem(AbstractItem): "deny_create": deny_create, "deny_read": deny_read, "hidden": hidden, - } \ No newline at end of file + } + + self.with_update = with_update + self.with_delete = with_delete \ No newline at end of file diff --git a/perxis/extensions/utils.py b/perxis/extensions/utils.py index b5ba8e6..d45242a 100644 --- a/perxis/extensions/utils.py +++ b/perxis/extensions/utils.py @@ -4,21 +4,31 @@ from perxis.extensions import manager_service_pb2 from perxis.extensions.item_models import DataSourceItem, SyncPolicyItem -def datasource_items_from_collections(collections_map: dict[str, str]) -> list[DataSourceItem]: +def datasource_items_from_collections( + collections_map: dict[str, str], + with_update: bool = True, + with_delete: bool = True, +) -> list[DataSourceItem]: """ Создание записей источников данных на базе маппинга коллекций """ return [ DataSourceItem( - collection_id=collection_id + collection_id=collection_id, + with_delete=with_delete, + with_update=with_update ) for collection_id in collections_map if collection_id ] -def sync_policies_from_collections(collections_map: dict[str, str]) -> list[SyncPolicyItem]: +def sync_policies_from_collections( + collections_map: dict[str, str], + with_update: bool = True, + with_delete: bool = True, +) -> list[SyncPolicyItem]: """ Создание записей синхронизации коллекций на базе маппинга коллекций """ @@ -27,6 +37,8 @@ def sync_policies_from_collections(collections_map: dict[str, str]) -> list[Sync SyncPolicyItem( collection_id=collection_id, name=collection_name, + with_update=with_update, + with_delete=with_delete, ) for collection_id, collection_name in collections_map.items() if collection_id -- GitLab From 7bd0b6ea290735a35b433b70d12ef156777f39be Mon Sep 17 00:00:00 2001 From: Maxim Podosochnyy <podosochnyy@perx.ru> Date: Wed, 26 Mar 2025 15:07:48 +0300 Subject: [PATCH 7/7] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D1=84=D0=BB=D0=B0=D0=B3=D0=B8=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20=D1=83=D0=BA=D0=B0=D0=B7=D0=B0=D0=BD=D0=B8=D1=8F?= =?UTF-8?q?=20=D0=B4=D0=BE=D0=BF=D1=83=D1=81=D0=B8=D0=BC=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B8=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D1=8F?= =?UTF-8?q?=20/=20=D1=83=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20item'?= =?UTF-8?q?=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/extension_service/servicer.py | 3 ++ perxis/extensions/extension_setup.py | 41 +++++++------------------- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/examples/extension_service/servicer.py b/examples/extension_service/servicer.py index f40984e..79aa7e5 100644 --- a/examples/extension_service/servicer.py +++ b/examples/extension_service/servicer.py @@ -87,6 +87,9 @@ class Servicer(ExtensionService): ), SyncPolicyItem( collection_id="test_collection", + # Запрет на изменение или удаление записи политики синхронизации + with_update=False, + with_delete=False, ), Item( collection_id="secrets", diff --git a/perxis/extensions/extension_setup.py b/perxis/extensions/extension_setup.py index 918f201..3bfa04e 100644 --- a/perxis/extensions/extension_setup.py +++ b/perxis/extensions/extension_setup.py @@ -584,19 +584,12 @@ class ExtensionSetup: ) for item in self.items: - try: - if item.rules: - all_rules_satisfied = all( - [ - await rule(item, space_id, env_id) - for rule - in item.rules - ] - ) + all_rules_satisfied = await item.all_rules_is_satisfied(space_id, env_id) - if not all_rules_satisfied: - continue + if not all_rules_satisfied: + continue + try: message = await wrapper.find( collection_id=item.collection_id, env_id=env_id, @@ -624,17 +617,10 @@ class ExtensionSetup: ) for item in self.items: - if item.rules: - all_rules_satisfied = all( - [ - await rule(item, space_id, env_id) - for rule - in item.rules - ] - ) + all_rules_satisfied = await item.all_rules_is_satisfied(space_id, env_id) - if not all_rules_satisfied: - continue + if not all_rules_satisfied: + continue try: message = await wrapper.find( @@ -714,17 +700,10 @@ class ExtensionSetup: ) for item in self.items: - if item.rules: - all_rules_satisfied = all( - [ - await rule(item, space_id, env_id) - for rule - in item.rules - ] - ) + all_rules_satisfied = await item.all_rules_is_satisfied(space_id, env_id) - if not all_rules_satisfied: - continue + if not all_rules_satisfied: + continue # Если установлен запрет на удаление item'ов if not item.with_delete: -- GitLab