From bcf32df44f39748ec57eda4115953c20b994e025 Mon Sep 17 00:00:00 2001 From: JimZhang Date: Tue, 18 Jul 2023 13:46:47 +0800 Subject: [PATCH] init project --- .gitignore | 160 ++++++++++++ LICENSE | 21 ++ README.md | 65 +++++ busrtworker/__init__.py | 2 + busrtworker/app.py | 277 +++++++++++++++++++++ busrtworker/boostrap.py | 95 +++++++ busrtworker/busrt/__init__.py | 3 + busrtworker/busrt/busrt_client.py | 274 ++++++++++++++++++++ busrtworker/busrt/busrt_rpc.py | 266 ++++++++++++++++++++ busrtworker/busrt/msgutils.py | 14 ++ busrtworker/kink/__init__.py | 2 + busrtworker/kink/__verstion__.py | 1 + busrtworker/kink/container.py | 97 ++++++++ busrtworker/kink/errors/__init__.py | 4 + busrtworker/kink/errors/conainer_error.py | 2 + busrtworker/kink/errors/execution_error.py | 5 + busrtworker/kink/errors/resolver_error.py | 5 + busrtworker/kink/errors/service_error.py | 5 + busrtworker/kink/inject.py | 245 ++++++++++++++++++ busrtworker/kink/py.typed | 0 busrtworker/scheduler/__init__.py | 124 +++++++++ busrtworker/tree.py | 166 ++++++++++++ example/api.py | 32 +++ poetry.lock | 247 ++++++++++++++++++ pyproject.toml | 43 ++++ tests/__init__.py | 0 26 files changed, 2155 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 busrtworker/__init__.py create mode 100644 busrtworker/app.py create mode 100644 busrtworker/boostrap.py create mode 100644 busrtworker/busrt/__init__.py create mode 100644 busrtworker/busrt/busrt_client.py create mode 100644 busrtworker/busrt/busrt_rpc.py create mode 100644 busrtworker/busrt/msgutils.py create mode 100644 busrtworker/kink/__init__.py create mode 100644 busrtworker/kink/__verstion__.py create mode 100644 busrtworker/kink/container.py create mode 100644 busrtworker/kink/errors/__init__.py create mode 100644 busrtworker/kink/errors/conainer_error.py create mode 100644 busrtworker/kink/errors/execution_error.py create mode 100644 busrtworker/kink/errors/resolver_error.py create mode 100644 busrtworker/kink/errors/service_error.py create mode 100644 busrtworker/kink/inject.py create mode 100644 busrtworker/kink/py.typed create mode 100644 busrtworker/scheduler/__init__.py create mode 100644 busrtworker/tree.py create mode 100644 example/api.py create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b0b6f3a --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9ebc1be --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 JimZhang (https://www.loom.run) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a7fda4 --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +# Busrt Worker +> `BUS/RT`是一个快速、灵活且非常易于使用的框架，它使用Rust/Tokio编写，受到NATS、ZeroMQ和Nanomsg的启发。 + +Busrt Worker是一个基于busrt消息中间件的异步框架，它对Python库进行了封装，使其更易于使用。 + + +## 特点 + +- 异步：Busrt Worker基于python asyncio异步引擎，可以轻松处理高并发请求。 +- 易用性：Busrt Worker对原生的busrt python客户端进行了封装，使用装饰器即可轻松创建rpc服务。 +- 高性能：BUS/RT 使用Rust/Tokio编写，具有出色的性能和可靠性。 + +## 用法 + +要使用Busrt Worker，请按照以下步骤操作： + +1. 安装Busrt Worker：`pip install busrt-worker` +2. 导入Busrt Worker：在您的Python项目中导入Busrt Worker。 +3. 创建Busrt Worker App：创建一个App对象，并注册连接信息。 +4. 处理消息：使用装饰器指明消息处理方式。 + +以下是一个示例代码片段，演示如何使用Busrt Worker： + +```python +import asyncio + +from loguru import logger + +from busrtworker import App, ConnectionInfo +# 创建 App对象 +app = App() + +# 连接信息 +api_ci = ConnectionInfo('api', 'localhost:9800', 'busrt.worker.test', static=True, topic='test/#') +caller_ci = ConnectionInfo('caller', 'localhost:9800', 'busrt.worker.test', static=True) + +# 注册连接 +api = app.registry(api_ci) +app.registry(caller_ci) + +# rpc调用 +@api.on_call() +def add(a, b): + return a + b + +# 主题订阅 +@api.subscribe('test/:name') +def print_name(name: str): + logger.info(f'{name} pub message') + +# 钩子函数注册 +@app.run_on_startup +async def test(server): + async def call(): + await asyncio.sleep(1) + logger.info(f'call remote add result {(await app.caller.add(api_ci.final_name,a=1, b=2))}') + await app.caller.send('test/i_am_caller') + asyncio.create_task(call()) + +# 启动程序 +app.run() +``` + +## License +The MIT License. \ No newline at end of file diff --git a/busrtworker/__init__.py b/busrtworker/__init__.py new file mode 100644 index 0000000..9374f8a --- /dev/null +++ b/busrtworker/__init__.py @@ -0,0 +1,2 @@ +from .app import App,ConnectionInfo + diff --git a/busrtworker/app.py b/busrtworker/app.py new file mode 100644 index 0000000..83d8ced --- /dev/null +++ b/busrtworker/app.py @@ -0,0 +1,277 @@ +import abc +import asyncio +import dataclasses +import functools +import signal +from collections import UserList +from enum import Enum, auto +from typing import Callable, Coroutine, Dict, List, Optional + +from loguru import logger + +from busrtworker.boostrap import RpcBoot +from busrtworker.busrt import OP_PUBLISH, Client, Frame, Rpc, serialize +from busrtworker.scheduler import ScheduledTaskRunner +from busrtworker.tree import RadixTree + + +@dataclasses.dataclass +class ConnectionInfo: + name: str = dataclasses.field() + uri: str = dataclasses.field() + client_prefix: str = dataclasses.field() + static: bool = dataclasses.field() + topic: str | None = dataclasses.field(default=None) + init_rpc: bool = dataclasses.field(default=True) + bus: Client = dataclasses.field(default=None, init=False) + rpc: Rpc = dataclasses.field(default=None, init=False) + final_name: str = dataclasses.field(default=None, init=False) + + def __getattr__(self, item): + if not self.init_rpc: + raise ValueError('must be init rpc client could call') + return getattr(self.rpc, item) + + async def send(self, topic, data=None, decode=True): + bus: Client = self.bus + await bus.send(topic, Frame(serialize(data) if decode else data, tp=OP_PUBLISH)) + + +class Router: + table: dict = {} + tree: RadixTree = RadixTree() + + def insert(self, path, handler, dynamic=False): + if not dynamic: + if path in self.table: + raise ValueError(f'conflict route {path}') + self.table[path] = handler + else: + self.tree.insert(path, handler, ['RPC']) + + def get(self, path): + if path in self.table: + return True, self.table[path], {} + return self.tree.get(path, 'RPC') + + +class ServiceEntrypoint: + def __init__(self, connection: ConnectionInfo, app: 'App'): + self.name = connection.name + self.app = app + if self.name not in self.app.callers: + self.app.callers[self.name] = {} + if self.name not in self.app.subscribes: + self.app.subscribes[self.name] = Router() + + def on_call(self, method=None, auto_decode=True, raw=False): + def _warp(f): + target = method or (f.func.__name__ if isinstance(f, functools.partial) else f.__name__) + self.app.callers[self.name][target] = (f, auto_decode, asyncio.iscoroutinefunction(f), raw) + return f + + return _warp + + def subscribe(self, topic, auto_decode=True, raw=False): + assert isinstance(topic, str), 'topic must be str or callable' + + def _warp(f): + self.app.subscribes[self.name].insert(topic, (f, auto_decode, asyncio.iscoroutinefunction(f), raw), + '/:' in topic) + return f + + return _warp + + +class Freezable(metaclass=abc.ABCMeta): + def __init__(self): + self._frozen = False + + @property + def frozen(self) -> bool: + return self._frozen + + async def freeze(self): + self._frozen = True + + +class Signal(UserList, asyncio.Event): + """ + Coroutine-based signal implementation tha behaves as an `asyncio.Event`. + + To connect a callback to a signal, use any list method. + + Signals are fired using the send() coroutine, which takes named + arguments. + """ + + def __init__(self, owner: Freezable) -> None: + UserList.__init__(self) + asyncio.Event.__init__(self) + self._owner = owner + self.frozen = False + + def __repr__(self): + return "".format( + self._owner, self.frozen, list(self) + ) + + async def send(self, *args, **kwargs): + """ + Sends data to all registered receivers. + """ + if self.frozen: + raise RuntimeError("Cannot send on frozen signal.") + + for receiver in self: + await receiver(*args, **kwargs) + + self.frozen = True + await self._owner.freeze() + self.set() + + +async def on_frame_default(app, frame): + logger.opt(lazy=True).debug('{x}', + x=lambda: f"default print 'Frame:', {hex(frame.type)}, frame.sender, frame.topic, frame.payload") + + +async def on_call_default(app, event): + logger.opt(lazy=True).debug('{x}', + x=lambda: f"default print 'Rpc:', {event.frame.sender}, {event.method}, {event.get_payload()}") + + +def entrypoint(f): + @functools.wraps(f) + def _(*args, **kwargs): + try: + loop = asyncio.get_event_loop() + return loop.run_until_complete(f(*args, **kwargs)) + except KeyboardInterrupt: + pass + + return _ + + +class AutoNameEnum(str, Enum): + def _generate_next_value_( # type: ignore + name: str, start: int, count: int, last_values: List[str] + ) -> str: + return name.lower() + + +class Options(AutoNameEnum): + MAX_CONCURRENCY = auto() + + +class DefaultValues: + RUN_EVERY_MAX_CONCURRENCY = 1 + + +class App(Freezable): + callers: dict[str, callable] = {} + subscribes: dict[str, Router] = {} + connections: Dict[str, ConnectionInfo] = {} + closeable = [] + on_frame_default: callable = on_frame_default + on_call_default: callable = on_call_default + task_runners = [] + + def __init__(self): + Freezable.__init__(self) + self.boot = RpcBoot() + self._on_startup: Signal = Signal(self) + self._on_shutdown: Signal = Signal(self) + self._on_startup.append(self.boot.startup) + self._on_shutdown.append(self.boot.shutdown) + signal.signal(signal.SIGINT, self.shutdown) + signal.signal(signal.SIGTERM, self.shutdown) + + def registry(self, connection: ConnectionInfo): + if self.frozen: + raise RuntimeError( + "You shouldn't change the state of a started application" + ) + self.connections[connection.name] = connection + return ServiceEntrypoint(connection, self) + + def set_on_frame_default(self, on_frame: callable): + self.on_frame_default = on_frame + + def set_on_call_default(self, on_call: callable): + self.on_call_default = on_call + + @entrypoint + async def run(self): + if self.frozen: + raise RuntimeError( + "You shouldn't change the state of a started application" + ) + logger.debug({"event": "Booting App..."}) + await self.startup() + + await self._on_shutdown.wait() + + async def startup(self): + """ + Causes on_startup signal + + Should be called in the event loop along with the request handler. + """ + await self._on_startup.send(self) + + def shutdown(self, *args) -> asyncio.Future: + """ + Schedules an on_startup signal + + Is called automatically when the application receives + a SIGINT or SIGTERM + """ + logger.debug('do shutdown') + return asyncio.ensure_future(self._on_shutdown.send(self)) + + def run_on_startup(self, coro: Callable[["App"], Coroutine]) -> None: + """ + Registers a coroutine to be awaited for during app startup + """ + self._on_startup.append(coro) + + def run_on_shutdown(self, coro: Callable[["App"], Coroutine]) -> None: + """ + Registers a coroutine to be awaited for during app shutdown + """ + self._on_shutdown.append(coro) + + def __getattr__(self, name): + return self.connections[name] + + def run_every(self, seconds: int, options: Optional[Dict] = None): + """ + Registers a coroutine to be called with a given interval + """ + if options is None: + options = {} + + max_concurrency = options.get( + Options.MAX_CONCURRENCY, DefaultValues.RUN_EVERY_MAX_CONCURRENCY + ) + + def wrapper(task: Callable[..., Coroutine]): + runner = ScheduledTaskRunner( + seconds=seconds, + task=task, + max_concurrency=max_concurrency, + ) + self._on_startup.append(runner.start) + self._on_shutdown.append(runner.stop) + self.task_runners.append(runner) + + return task + + return wrapper + + def rpc_running(self, name): + if connection := self.connections.get(name, None): + if rpc := connection.rpc: + return rpc.is_connected() + return False diff --git a/busrtworker/boostrap.py b/busrtworker/boostrap.py new file mode 100644 index 0000000..c52bffb --- /dev/null +++ b/busrtworker/boostrap.py @@ -0,0 +1,95 @@ +import asyncio +import functools +import os +import traceback +from typing import TYPE_CHECKING + +from .busrt import Client, Rpc, RpcException, deserialize, on_frame_default, serialize + +if TYPE_CHECKING: # pragma: no cover + from .app import App # noqa: F401 + + +class RpcBoot: + async def startup(self, app: 'App'): + for i, (name, connection) in enumerate(app.connections.items()): + uri = connection.uri + static = connection.static + init_rpc = connection.init_rpc + prefix = connection.client_prefix + topic = connection.topic + connection.final_name = f'{prefix}.{name}' if static else f'{prefix}.{os.urandom(4).hex()}' + bus = Client(uri, connection.final_name) + connection.bus = bus + + app.closeable.append(bus.disconnect) + await bus.connect() + if topic: + await bus.subscribe(topic) + + async def on_frame(frame, router): + success, route, params = router.get(frame.topic) + if not success: + return await app.on_frame_default(frame) + else: + func, auto_decode, is_async, raw = route + if auto_decode: + payload = deserialize(frame.payload) + if raw: + payload['frame'] = frame + if is_async: + return await func(**payload, **params) + return func(**payload, **params) + else: + if is_async: + return await func(frame, **params) + return func(frame, **params) + + on_frame_func = functools.partial(on_frame, router=app.subscribes.get(name, None)) + else: + on_frame_func = on_frame_default + if init_rpc: + + async def on_call(event, caller): + try: + method = event.method.decode('utf-8') + caller_tuple = caller.get(method, None) + if not caller_tuple: + return serialize(await app.on_call_default(event)) + call, auto_decode, is_async, raw = caller_tuple + if auto_decode: + payload = deserialize(event.get_payload()) + if raw: + payload['event'] = event + if is_async: + return serialize(await call(**payload)) + return serialize(call(**payload)) + else: + if is_async: + return serialize(await call(event)) + return serialize(call(event)) + except Exception as e: + traceback.print_exc() + raise RpcException(str(e), 11) + + rpc_client = Rpc(bus) + connection.rpc = rpc_client + if topic: + rpc_client.on_frame = functools.partial(on_frame_func, router=app.subscribes.get(name, None)) + rpc_client.on_call = functools.partial(on_call, caller=app.callers.get(name, None)) + elif topic: + bus.on_frame = functools.partial(on_frame_func, router=app.subscribes.get(name, None)) + + async def shutdown(self, app: 'App'): + gathers = [] + + for func in app.closeable: + if asyncio.iscoroutinefunction(func): + gathers.append(func()) + else: + try: + func() + except: + traceback.print_exc() + if gathers: + await asyncio.gather(*gathers, return_exceptions=True) diff --git a/busrtworker/busrt/__init__.py b/busrtworker/busrt/__init__.py new file mode 100644 index 0000000..cd127e2 --- /dev/null +++ b/busrtworker/busrt/__init__.py @@ -0,0 +1,3 @@ +from .busrt_client import * # noqa: F403 +from .busrt_rpc import * # noqa: F403 +from .msgutils import * # noqa: F403 \ No newline at end of file diff --git a/busrtworker/busrt/busrt_client.py b/busrtworker/busrt/busrt_client.py new file mode 100644 index 0000000..dcfb73f --- /dev/null +++ b/busrtworker/busrt/busrt_client.py @@ -0,0 +1,274 @@ +import asyncio +from loguru import logger + +GREETINGS = 0xEB + +PROTOCOL_VERSION = 1 + +OP_NOP = 0 +OP_PUBLISH = 1 +OP_SUBSCRIBE = 2 +OP_UNSUBSCRIBE = 3 +OP_MESSAGE = 0x12 +OP_BROADCAST = 0x13 +OP_ACK = 0xFE + +RESPONSE_OK = 0x01 +ERR_CLIENT_NOT_REGISTERED = 0x71 +ERR_DATA = 0x72 +ERR_IO = 0x73 +ERR_OTHER = 0x74 +ERR_NOT_SUPPORTED = 0x75 +ERR_BUSY = 0x76 +ERR_NOT_DELIVERED = 0x77 +ERR_TIMEOUT = 0x78 + +PING_FRAME = b'\x00' * 9 + + +async def on_frame_default(frame): + pass + + +class Client: + + def __init__(self, path, name): + self.path = path + self.writer = None + self.reader_fut = None + self.pinger_fut = None + self.buf_size = 8192 + self.name = name + self.frame_id = 0 + self.ping_interval = 1 + self.on_frame = on_frame_default + self.socket_lock = asyncio.Lock() + self.mgmt_lock = asyncio.Lock() + self.connected = False + self.frames = {} + self.timeout = 1 + + async def connect(self): + async with self.mgmt_lock: + if self.path.endswith('.sock') or self.path.endswith( + '.socket') or self.path.endswith( + '.ipc') or self.path.startswith('/'): + if hasattr(asyncio, 'open_unix_connection'): + reader, writer = await asyncio.open_unix_connection( + self.path, limit=self.buf_size) + else: + raise ValueError('only support unix like system') + else: + host, port = self.path.rsplit(':', maxsplit=2) + reader, writer = await asyncio.open_connection( + host, int(port), limit=self.buf_size) + buf = await asyncio.wait_for(self.readexactly(reader, 3), + timeout=self.timeout) + if buf[0] != GREETINGS: + raise RuntimeError('Unsupported protocol') + if int.from_bytes(buf[1:3], 'little') != PROTOCOL_VERSION: + raise RuntimeError('Unsupported protocol version') + writer.write(buf) + await asyncio.wait_for(writer.drain(), timeout=self.timeout) + buf = await asyncio.wait_for(self.readexactly(reader, 1), + timeout=self.timeout) + if buf[0] != RESPONSE_OK: + raise RuntimeError(f'Server response: {hex(buf[0])}') + name = self.name.encode() + writer.write(len(name).to_bytes(2, 'little') + name) + await asyncio.wait_for(writer.drain(), timeout=self.timeout) + buf = await asyncio.wait_for(self.readexactly(reader, 1), + timeout=self.timeout) + if buf[0] != RESPONSE_OK: + raise RuntimeError(f'Server response: {hex(buf[0])}') + self.writer = writer + self.connected = True + self.pinger_fut = asyncio.ensure_future(self._t_pinger()) + self.reader_fut = asyncio.ensure_future(self._t_reader(reader)) + + async def handle_daemon_exception(self, e): + async with self.mgmt_lock: + if self.connected: + await self._disconnect() + import traceback + logger.error(traceback.format_exc()) + + async def _t_pinger(self): + try: + while True: + await asyncio.sleep(self.ping_interval) + async with self.socket_lock: + self.writer.write(PING_FRAME) + await asyncio.wait_for(self.writer.drain(), + timeout=self.timeout) + except Exception as e: + asyncio.ensure_future(self.handle_daemon_exception(e)) + + async def _t_reader(self, reader): + try: + while True: + buf = await self.readexactly(reader, 6) + if buf[0] == OP_NOP: + continue + elif buf[0] == OP_ACK: + op_id = int.from_bytes(buf[1:5], 'little') + try: + o = self.frames.pop(op_id) + o.result = buf[5] + o.completed.set() + except KeyError: + logger.warning(f'orphaned BUS/RT frame ack {op_id}') + else: + + async def read_frame(tp, data_len): + frame = Frame() + frame.type = tp + sender = await reader.readuntil(b'\x00') + data_len -= len(sender) + frame.sender = sender[:-1].decode() + if buf[0] == OP_PUBLISH: + topic = await reader.readuntil(b'\x00') + data_len -= len(topic) + frame.topic = topic[:-1].decode() + else: + frame.topic = None + data = b'' + while len(data) < data_len: + buf_size = data_len - len(data) + len(data) + try: + data += await reader.readexactly( + buf_size if buf_size < self.buf_size else + self.buf_size) + except asyncio.IncompleteReadError: + pass + frame.payload = data + return frame + + try: + data_len = int.from_bytes(buf[1:5], 'little') + frame = await read_frame(buf[0], data_len) + except Exception as e: + logger.error(f'Invalid frame from the server: {e}') + raise + asyncio.ensure_future(self.on_frame(frame)) + except Exception as e: + asyncio.ensure_future(self.handle_daemon_exception(e)) + + async def readexactly(self, reader, data_len): + data = b'' + while len(data) < data_len: + buf_size = data_len - len(data) + try: + chunk = await reader.readexactly( + buf_size if buf_size < self.buf_size else self.buf_size) + data += chunk + except asyncio.IncompleteReadError: + await asyncio.sleep(0.01) + return data + + async def disconnect(self): + async with self.mgmt_lock: + await self._disconnect() + + async def _disconnect(self): + self.connected = False + self.writer.close() + if self.reader_fut is not None: + self.reader_fut.cancel() + if self.pinger_fut is not None: + self.pinger_fut.cancel() + + async def send(self, target=None, frame=None): + try: + async with self.socket_lock: + self.frame_id += 1 + if self.frame_id > 0xffff_ffff: + self.frame_id = 1 + frame_id = self.frame_id + o = ClientFrame(frame.qos) + if frame.qos & 0b1 != 0: + self.frames[frame_id] = o + flags = frame.type | frame.qos << 6 + if frame.type == OP_SUBSCRIBE or frame.type == OP_UNSUBSCRIBE: + topics = frame.topic if isinstance(frame.topic, + list) else [frame.topic] + payload = b'\x00'.join(t.encode() for t in topics) + self.writer.write( + frame_id.to_bytes(4, 'little') + + flags.to_bytes(1, 'little') + + len(payload).to_bytes(4, 'little') + payload) + else: + frame_len = len(target) + len(frame.payload) + 1 + if frame.header is not None: + frame_len += len(frame.header) + if frame_len > 0xffff_ffff: + raise ValueError('frame too large') + self.writer.write( + frame_id.to_bytes(4, 'little') + + flags.to_bytes(1, 'little') + + frame_len.to_bytes(4, 'little') + target.encode() + + b'\x00' + + (frame.header if frame.header is not None else b'')) + self.writer.write(frame.payload.encode( + ) if isinstance(frame.payload, str) else frame.payload) + await self.writer.drain() + return o + except: + try: + del self.frames[frame_id] + except KeyError: + pass + raise + + def subscribe(self, topics): + frame = Frame(tp=OP_SUBSCRIBE) + frame.topic = topics + return self.send(None, frame) + + def unsubscribe(self, topics): + frame = Frame(tp=OP_UNSUBSCRIBE) + frame.topic = topics + return self.send(None, frame) + + def is_connected(self): + return self.connected + + +class ClientFrame: + + def __init__(self, qos): + self.qos = qos + self.result = 0 + if qos & 0b1 != 0: + self.completed = asyncio.Event() + else: + self.completed = None + + def is_completed(self): + if self.qos & 0b1 != 0: + return self.completed.is_set() + else: + return True + + async def wait_completed(self, timeout=None): + if self.qos & 0b1 == 0: + return RESPONSE_OK + elif timeout: + await asyncio.wait_for(self.completed.wait(), timeout=timeout) + else: + await self.completed.wait() + return self.result + + +class Frame: + + def __init__(self, payload=None, tp=OP_MESSAGE, qos=0): + self.payload = payload + # used for zero-copy + self.header = None + self.type = tp + self.qos = qos + + +__all__ = ["Client","Frame","OP_PUBLISH","OP_MESSAGE","OP_BROADCAST","ERR_IO","ERR_TIMEOUT","RESPONSE_OK","on_frame_default"] \ No newline at end of file diff --git a/busrtworker/busrt/busrt_rpc.py b/busrtworker/busrt/busrt_rpc.py new file mode 100644 index 0000000..9791f6c --- /dev/null +++ b/busrtworker/busrt/busrt_rpc.py @@ -0,0 +1,266 @@ +import asyncio +import functools + +from loguru import logger + +from .busrt_client import ( + ERR_IO, + ERR_TIMEOUT, + OP_MESSAGE, + RESPONSE_OK, + on_frame_default, +) +from .msgutils import deserialize, serialize + +RPC_NOTIFICATION_HEADER = b'\x00' +RPC_REQUEST_HEADER = b'\x01' +RPC_REPLY_HEADER = b'\x11' +RPC_ERROR_REPLY_HEADER = b'\x12' + +RPC_NOTIFICATION = 0x00 +RPC_REQUEST = 0x01 +RPC_REPLY = 0x11 +RPC_ERROR = 0x12 + +RPC_ERROR_CODE_PARSE = -32700 +RPC_ERROR_CODE_INVALID_REQUEST = -32600 +RPC_ERROR_CODE_METHOD_NOT_FOUND = -32601 +RPC_ERROR_CODE_INVALID_METHOD_PARAMS = -32602 +RPC_ERROR_CODE_INTERNAL = -32603 + + +async def on_call_default(event): + raise RpcException('RPC Engine not initialized', + RPC_ERROR_CODE_METHOD_NOT_FOUND) + + +async def on_notification_default(event): + pass + + +def format_rpc_e_msg(e): + if isinstance(e, RpcException): + return e.rpc_error_payload + else: + return str(e) + + +class RpcException(Exception): + + def __init__(self, msg: str | bytes = '', code=RPC_ERROR_CODE_INTERNAL): + self.rpc_error_code = code + self.rpc_error_payload = msg + super().__init__(msg if isinstance(msg, str) else msg.decode()) + + def __str__(self): + return super().__str__() + f' (code: {self.rpc_error_code})' + + +class Rpc: + + def __init__(self, client): + self.client = client + self.client.on_frame = self._handle_frame + self.call_id = 0 + self.call_lock = asyncio.Lock() + self.calls = {} + self.on_frame = on_frame_default + self.on_call = on_call_default + self.on_notification = on_notification_default + + def is_connected(self): + return self.client.connected + + def notify(self, target, notification): + return self.client.send(target, notification) + + def call0(self, target, request): + request.header = RPC_REQUEST_HEADER + b'\x00\x00\x00\x00' + \ + request.method + b'\x00' + return self.client.send(target, request) + + async def call(self, target, request): + async with self.call_lock: + call_id = self.call_id + 1 + if call_id == 0xffff_ffff: + self.call_id = 0 + else: + self.call_id = call_id + call_event = RpcCallEvent() + self.calls[call_id] = call_event + request.header = RPC_REQUEST_HEADER + call_id.to_bytes( + 4, 'little') + request.method + b'\x00' + try: + try: + code = await (await self.client.send( + target, + request)).wait_completed(timeout=self.client.timeout) + if code != RESPONSE_OK: + try: + del self.calls[call_id] + except KeyError: + pass + err_code = -32000 - code + call_event.error = RpcException('RPC error', code=err_code) + call_event.completed.set() + except asyncio.TimeoutError: + try: + del self.calls[call_id] + except KeyError: + pass + err_code = -32000 - ERR_TIMEOUT + call_event.error = RpcException('RPC timeout', code=err_code) + call_event.completed.set() + except Exception as e: + try: + del self.calls[call_id] + except KeyError: + pass + call_event.error = RpcException(str(e), code=-32000 - ERR_IO) + call_event.completed.set() + return call_event + + def __getattr__(self, method): + return functools.partial(self.params_call, method=method) + + async def params_call(self, target, method: str, **kwargs): + params = kwargs + if '_raw' in kwargs: + params = kwargs.pop('_raw') + params = params or None + c0 = False + if method.endswith('0'): + c0 = True + method = method[:-1] + request = Request(method, serialize(params)) + if c0: + await self.call0(target, request) + return None + result = await self.call(target, request) + return deserialize((await result.wait_completed()).get_payload()) + + async def _handle_frame(self, frame): + try: + if frame.type == OP_MESSAGE: + if frame.payload[0] == RPC_NOTIFICATION: + event = Event(RPC_NOTIFICATION, frame, 1) + await self.on_notification(event) + elif frame.payload[0] == RPC_REQUEST: + sender = frame.sender + call_id_b = frame.payload[1:5] + call_id = int.from_bytes(call_id_b, 'little') + method = frame.payload[5:5 + + frame.payload[5:].index(b'\x00')] + event = Event(RPC_REQUEST, frame, 6 + len(method)) + event.call_id = call_id + event.method = method + if call_id == 0: + await self.on_call(event) + else: + reply = Reply() + try: + reply.payload = await self.on_call(event) + if reply.payload is None: + reply.payload = b'' + reply.header = RPC_REPLY_HEADER + call_id_b + except Exception as e: + code = getattr(e, 'rpc_error_code', RPC_ERROR_CODE_INTERNAL) + + reply.header = ( + RPC_ERROR_REPLY_HEADER + call_id_b + + code.to_bytes(2, 'little', signed=True)) + reply.payload = format_rpc_e_msg(e) + await self.client.send(sender, reply) + elif frame.payload[0] == RPC_REPLY or frame.payload[ + 0] == RPC_ERROR: + call_id = int.from_bytes(frame.payload[1:5], 'little') + try: + call_event = self.calls.pop(call_id) + call_event.frame = frame + if frame.payload[0] == RPC_ERROR: + err_code = int.from_bytes(frame.payload[5:7], + 'little', + signed=True) + call_event.error = RpcException(frame.payload[7:], + code=err_code) + call_event.completed.set() + except KeyError: + logger.warning(f'orphaned RPC response: {call_id}') + else: + logger.error(f'Invalid RPC frame code: {frame.payload[0]}') + else: + await self.on_frame(frame) + except Exception: + import traceback + logger.error(traceback.format_exc()) + + +class RpcCallEvent: + + def __init__(self): + self.frame = None + self.error = None + self.completed = asyncio.Event() + + def is_completed(self): + return self.completed.is_set() + + async def wait_completed(self, timeout=None): + if timeout: + if not await asyncio.wait_for(self.completed.wait(), + timeout=timeout): + raise TimeoutError + else: + await self.completed.wait() + if self.error is None: + return self + else: + raise self.error + + def get_payload(self): + return self.frame.payload[5:] + + def is_empty(self): + return len(self.frame.payload) <= 5 + + +class Event: + + def __init__(self, tp, frame, payload_pos): + self.tp = tp + self.frame = frame + self._payload_pos = payload_pos + + def get_payload(self): + return self.frame.payload[self._payload_pos:] + + +class Notification: + + def __init__(self, payload=b''): + self.payload = payload + self.type = OP_MESSAGE + self.qos = 1 + self.header = RPC_NOTIFICATION_HEADER + + +class Request: + + def __init__(self, method, params=b''): + self.payload = b'' if params is None else params + self.type = OP_MESSAGE + self.qos = 1 + self.method = method.encode() if isinstance(method, str) else method + self.header = None + + +class Reply: + + def __init__(self, result=b''): + self.payload = result + self.type = OP_MESSAGE + self.qos = 1 + self.header = None + + +__all__ = ["Rpc", "Request", "Reply", "RpcCallEvent", "Event", "RpcException"] diff --git a/busrtworker/busrt/msgutils.py b/busrtworker/busrt/msgutils.py new file mode 100644 index 0000000..ca03119 --- /dev/null +++ b/busrtworker/busrt/msgutils.py @@ -0,0 +1,14 @@ +import msgpack +import zstd + + +def deserialize(data): + if data is None: + return None + return msgpack.unpackb(zstd.decompress(data)) + + +def serialize(data): + if data is None: + data={} + return zstd.compress(msgpack.dumps(data)) diff --git a/busrtworker/kink/__init__.py b/busrtworker/kink/__init__.py new file mode 100644 index 0000000..be25dfe --- /dev/null +++ b/busrtworker/kink/__init__.py @@ -0,0 +1,2 @@ +from .container import * +from .inject import * diff --git a/busrtworker/kink/__verstion__.py b/busrtworker/kink/__verstion__.py new file mode 100644 index 0000000..4c513f3 --- /dev/null +++ b/busrtworker/kink/__verstion__.py @@ -0,0 +1 @@ +__version__ = "0.6.6" diff --git a/busrtworker/kink/container.py b/busrtworker/kink/container.py new file mode 100644 index 0000000..05b137e --- /dev/null +++ b/busrtworker/kink/container.py @@ -0,0 +1,97 @@ +from types import LambdaType +from typing import Any, Callable, Dict, List, Type, Union + +from .errors.service_error import ServiceError + +_MISSING_SERVICE = object() + + +class Container: + def __init__(self): + self._memoized_services: Dict[Union[str, Type], Any] = {} + self._services: Dict[Union[str, Type], Any] = {} + self._factories: Dict[Union[str, Type], Callable[[Container], Any]] = {} + self._aliases: Dict[Union[str, Type], List[Union[str, Type]]] = {} + + def __setitem__(self, key: Union[str, Type], value: Any) -> None: + self._services[key] = value + + if key in self._memoized_services: + del self._memoized_services[key] + + def add_alias(self, name: Union[str, Type], target: Union[str, Type]): + if List[target] in self._memoized_services: # type: ignore + del self._memoized_services[List[target]] # type: ignore + + if name not in self._aliases: + self._aliases[name] = [] + self._aliases[name].append(target) + + def __getitem__(self, key: Union[str, Type]) -> Any: + if key in self._factories: + return self._factories[key](self) + + service = self._get(key) + + if service is not _MISSING_SERVICE: + return service + + if key in self._aliases: + unaliased_key = self._aliases[key][0] # By default return first aliased service + if unaliased_key in self._factories: + return self._factories[unaliased_key](self) + service = self._get(unaliased_key) + # service = self._get(self._aliases[key][0]) # By default return first aliased service + + if service is not _MISSING_SERVICE: + return service + + # Support aliasing + if self._has_alias_list_for(key): + result = [self._get(alias) for alias in self._aliases[key.__args__[0]]] # type: ignore + self._memoized_services[key] = result + return result + + raise ServiceError(f"Service {key} is not registered.") + + def _get(self, key: Union[str, Type]) -> Any: + if key in self._memoized_services: + return self._memoized_services[key] + + if key not in self._services: + return _MISSING_SERVICE + + value = self._services[key] + + if isinstance(value, LambdaType) and value.__name__ == "": + self._memoized_services[key] = value(self) + return self._memoized_services[key] + + return value + + def __contains__(self, key) -> bool: + contains = key in self._services or key in self._factories or key in self._aliases + + if contains: + return contains + + if self._has_alias_list_for(key): + return True + + return False + + def _has_alias_list_for(self, key: Union[str, Type]) -> bool: + return hasattr(key, "__origin__") and hasattr(key, "__args__") and key.__origin__ == list and key.__args__[0] in self._aliases # type: ignore + + @property + def factories(self) -> Dict[Union[str, Type], Callable[["Container"], Any]]: + return self._factories + + def clear_cache(self) -> None: + self._memoized_services = {} + + +di: Container = Container() + + +__all__ = ["Container", "di"] diff --git a/busrtworker/kink/errors/__init__.py b/busrtworker/kink/errors/__init__.py new file mode 100644 index 0000000..4933f32 --- /dev/null +++ b/busrtworker/kink/errors/__init__.py @@ -0,0 +1,4 @@ +from .conainer_error import ContainerError +from .execution_error import ExecutionError +from .resolver_error import ResolverError +from .service_error import ServiceError diff --git a/busrtworker/kink/errors/conainer_error.py b/busrtworker/kink/errors/conainer_error.py new file mode 100644 index 0000000..aa90b4a --- /dev/null +++ b/busrtworker/kink/errors/conainer_error.py @@ -0,0 +1,2 @@ +class ContainerError(RuntimeError): + pass diff --git a/busrtworker/kink/errors/execution_error.py b/busrtworker/kink/errors/execution_error.py new file mode 100644 index 0000000..2d443a9 --- /dev/null +++ b/busrtworker/kink/errors/execution_error.py @@ -0,0 +1,5 @@ +from .conainer_error import ContainerError + + +class ExecutionError(ContainerError): + pass diff --git a/busrtworker/kink/errors/resolver_error.py b/busrtworker/kink/errors/resolver_error.py new file mode 100644 index 0000000..12bf5f5 --- /dev/null +++ b/busrtworker/kink/errors/resolver_error.py @@ -0,0 +1,5 @@ +from .conainer_error import ContainerError + + +class ResolverError(ContainerError): + pass diff --git a/busrtworker/kink/errors/service_error.py b/busrtworker/kink/errors/service_error.py new file mode 100644 index 0000000..0667bf5 --- /dev/null +++ b/busrtworker/kink/errors/service_error.py @@ -0,0 +1,5 @@ +from .conainer_error import ContainerError + + +class ServiceError(ContainerError, KeyError): + pass diff --git a/busrtworker/kink/inject.py b/busrtworker/kink/inject.py new file mode 100644 index 0000000..ba7d984 --- /dev/null +++ b/busrtworker/kink/inject.py @@ -0,0 +1,245 @@ +import asyncio +import sys +from abc import ABC +from functools import wraps +from inspect import Parameter as InspectParameter, isclass, signature +from typing import Any, Callable, Dict, ForwardRef, NewType, Tuple, Type, TypeVar, Union # type: ignore + +from typing_extensions import Protocol + +from .container import Container, di +from .errors import ExecutionError + +T = TypeVar("T") +S = TypeVar("S") + +ServiceDefinition = Union[Type[S], Callable] +ServiceResult = Union[S, Callable] + +Undefined = NewType("Undefined", int) + + +class _ProtocolInit(Protocol): + pass + + +_no_init = _ProtocolInit.__init__ + + +def _resolve_forward_reference(module: Any, ref: Union[str, ForwardRef]) -> Any: + if isinstance(ref, str): + name = ref + else: + name = ref.__forward_arg__ + + if name in sys.modules[module].__dict__: + return sys.modules[module].__dict__[name] + + return None + + +class Parameter: + type: Any + name: str + default: Any + + def __init__(self, name: str, type: Any = Any, default: Any = Undefined): + self.name = name + self.type = type + self.default = default + + +def _inspect_function_arguments( + function: Callable, +) -> Tuple[Tuple[str, ...], Dict[str, Parameter]]: + parameters_name: Tuple[str, ...] = tuple(signature(function).parameters.keys()) + parameters = {} + + for name, parameter in signature(function).parameters.items(): + + if isinstance(parameter.annotation, (str, ForwardRef)) and hasattr(function, "__module__"): + annotation = _resolve_forward_reference(function.__module__, parameter.annotation) + else: + annotation = parameter.annotation + + parameters[name] = Parameter( + parameter.name, + annotation, + parameter.default if parameter.default is not InspectParameter.empty else Undefined, + ) + + return parameters_name, parameters + + +def _resolve_function_kwargs( + alias_map: Dict[str, str], + parameters_name: Tuple[str, ...], + parameters: Dict[str, Parameter], + container: Container, +) -> Dict[str, Any]: + resolved_kwargs = {} + for name in parameters_name: + if name in alias_map and alias_map[name] in container: + resolved_kwargs[name] = container[alias_map[name]] + continue + + if name in container: + resolved_kwargs[name] = container[name] + continue + + if parameters[name].type in container: + resolved_kwargs[name] = container[parameters[name].type] + continue + + if parameters[name].default is not Undefined: + resolved_kwargs[name] = parameters[name].default + + return resolved_kwargs + + +def _decorate(binding: Dict[str, Any], service: ServiceDefinition, container: Container) -> ServiceResult: + # ignore abstract class initialiser and protocol initialisers + if ( + service in [ABC.__init__, _no_init] or service.__name__ == "_no_init" + ): # FIXME: fix this when typing_extensions library gets fixed + return service + + # Add class definition to dependency injection + parameters_name, parameters = _inspect_function_arguments(service) + + def _resolve_kwargs(args, kwargs) -> dict: + # attach named arguments + passed_kwargs = {**kwargs} + + # resolve positional arguments + if args: + for key, value in enumerate(args): + passed_kwargs[parameters_name[key]] = value + + # prioritise passed kwargs and args resolving + if len(passed_kwargs) == len(parameters_name): + return passed_kwargs + + resolved_kwargs = _resolve_function_kwargs(binding, parameters_name, parameters, container) + + all_kwargs = {**resolved_kwargs, **passed_kwargs} + + if len(all_kwargs) < len(parameters_name): + missing_parameters = [arg for arg in parameters_name if arg not in all_kwargs] + raise ExecutionError( + "Cannot execute function without required parameters. " + + f"Did you forget to bind the following parameters: `{'`, `'.join(missing_parameters)}`?" + ) + + return all_kwargs + + @wraps(service) + def _decorated(*args, **kwargs): + # all arguments were passed + if len(args) == len(parameters_name): + return service(*args, **kwargs) + + if parameters_name == tuple(kwargs.keys()): + return service(**kwargs) + + all_kwargs = _resolve_kwargs(args, kwargs) + return service(**all_kwargs) + + @wraps(service) + async def _async_decorated(*args, **kwargs): + # all arguments were passed + if len(args) == len(parameters_name): + return await service(*args) + + if parameters_name == tuple(kwargs.keys()): + return await service(**kwargs) + + all_kwargs = _resolve_kwargs(args, kwargs) + return await service(**all_kwargs) + + if asyncio.iscoroutinefunction(service): + return _async_decorated + + return _decorated + + +def inject( + _service: ServiceDefinition = None, + alias: Any = None, + bind: Dict[str, Any] = None, + container: Container = di, + use_factory: bool = False, +) -> Union[ServiceResult, Callable[[ServiceDefinition], ServiceResult]]: + def _decorator(_service: ServiceDefinition) -> ServiceResult: + if isclass(_service): + setattr( + _service, + "__init__", + _decorate(bind or {}, getattr(_service, "__init__"), container), + ) + if use_factory: + container.factories[_service] = lambda _di: _service() + if alias: + container.add_alias(alias, _service) + else: + container[_service] = lambda _di: _service() + if alias: + container.add_alias(alias, _service) + + return _service + + service_function = _decorate(bind or {}, _service, container) + container[service_function.__name__] = service_function + if alias: + container.add_alias(alias, service_function.__name__) + + return service_function + + if _service is None: + return _decorator + + return _decorator(_service) + + +def provider( + _service: ServiceDefinition = None, + alias: Any = None, + bind: Dict[str, Any] = None, + container: Container = di, + process: bool = False, +) -> Union[ServiceResult, Callable[[ServiceDefinition], ServiceResult]]: + def _decorator(_service: ServiceDefinition) -> ServiceResult: + if isclass(_service): + setattr( + _service, + "__init__", + _decorate(bind or {}, getattr(_service, "__init__"), container), + ) + if process: + container.factories[_service] = lambda _di: _service() + if alias: + container.add_alias(alias, _service) + else: + container[_service] = lambda _di: _service() + if alias: + container.add_alias(alias, _service) + + return _service + + service_function = _decorate(bind or {}, _service, container) + if process: + container.factories[service_function.__name__] = lambda x: service_function() + else: + container[service_function.__name__] = lambda x: service_function() + if alias: + container.add_alias(alias, service_function.__name__) + + return service_function + + if _service is None: + return _decorator + + return _decorator(_service) + + +__all__ = ["inject", "provider"] diff --git a/busrtworker/kink/py.typed b/busrtworker/kink/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/busrtworker/scheduler/__init__.py b/busrtworker/scheduler/__init__.py new file mode 100644 index 0000000..a22cc34 --- /dev/null +++ b/busrtworker/scheduler/__init__.py @@ -0,0 +1,124 @@ +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any, Awaitable, Callable, Coroutine, Optional, Set, Union + +AsyncFuncType = Callable[[Any, Any], Awaitable[Any]] +class ClockTicker(AsyncIterator): + """ + T - A clock tick + F - Something that happens inside an iteration ("x" = running "-" = waiting) + I - A clock iteration + + E.g: + + async for tick in Clock(seconds=2): + await asyncio.sleep(3) + + + T: 15------17------19------21------23------25------27------29------ + F: xxxxxxxxxxxxx---xxxxxxxxxxxxx---xxxxxxxxxxxxx---xxxxxxxxxxxxx--- + I: x---------------x---------------x---------------x--------------- + + """ + + def __init__(self, seconds: Union[float, int]) -> None: + """ + :param seconds: Tick interval in seconds + """ + self.seconds = seconds + self.current_iteration = 0 + self._tick_event = asyncio.Event() + self._running: Optional[bool] = None + self._main_task: Optional[asyncio.Future] = None + + def __aiter__(self) -> AsyncIterator: + if self._running is not None: + raise RuntimeError("Cannot reuse a clock instance.") + + self._running = True + self._main_task = asyncio.ensure_future(self._run()) + return self + + async def __anext__(self) -> int: + if not self._running: + raise StopAsyncIteration + + self._tick_event.clear() + await self._tick_event.wait() + + i = self.current_iteration + self.current_iteration += 1 + return i + + async def _run(self) -> None: + while self._running: + self._tick_event.set() + await asyncio.sleep(self.seconds) + self._tick_event.clear() + + async def stop(self) -> None: + self._running = False + if self._main_task: + await self._main_task + + +def perf_counter_ms() -> float: + """ + Return the value (in fractional milliseconds) of a performance counter, + i.e. a clock with the highest available resolution to measure a short + duration. It does include time elapsed during sleep and is system-wide. + The reference point of the returned value is undefined, so that only the + difference between the results of consecutive calls is valid. + """ + return time.perf_counter() * 1000 + + +class ScheduledTaskRunner: + def __init__( + self, + seconds: int, + task: Callable[[], Coroutine], + max_concurrency: int, + ) -> None: + self.seconds = seconds + self.max_concurrency = max_concurrency + self.task = task + self.running_tasks: Set[asyncio.Future] = set() + self.task_is_done_event = asyncio.Event() + self._started = False + self.clock = ClockTicker(seconds=self.seconds) + + async def can_dispatch_task(self) -> bool: + if len(self.running_tasks) < self.max_concurrency: + return True + + if await self.task_is_done_event.wait(): + return True + return False + + async def _wrapped_task(self) -> None: + """ + Wraps the future task on a coroutine that's responsible for unregistering + itself from the "running tasks" and emitting an "task is done" event + """ + try: + await self.task() + finally: + self.task_is_done_event.set() + self.running_tasks.remove(asyncio.current_task()) # type: ignore + + async def start(self,*args,**kwargs) -> asyncio.Future: + self._started = True + return asyncio.ensure_future(self._run()) + + async def stop(self,*args,**kwargs) -> None: + await self.clock.stop() + await asyncio.gather(*self.running_tasks) + + async def _run(self) -> None: + async for _ in self.clock: + if await self.can_dispatch_task(): + task = asyncio.ensure_future(self._wrapped_task()) + self.running_tasks.add(task) + self.task_is_done_event.clear() diff --git a/busrtworker/tree.py b/busrtworker/tree.py new file mode 100644 index 0000000..26b5454 --- /dev/null +++ b/busrtworker/tree.py @@ -0,0 +1,166 @@ +class RadixTreeNode(object): + def __init__(self, path=None, handler=None, methods=None): + self.path = path + self.methods = {} + self.children = [] + self.indices = str() + self.size = 0 + + self.add_methods(methods, handler) + + def __repr__(self): + return (''.format(self.path, self.methods, self.indices, + self.children)) + + def add_methods(self, methods, handler): + if not methods: + return + + if not isinstance(methods, (list, tuple, set)): + methods = [methods] + + for method in methods: + if method in self.methods and self.methods[method] != handler: + raise KeyError( + '{} conflicts with existed handler ' + '{}'.format(handler, self.methods[method])) + + self.methods[method] = handler + + def bisect(self, target): + low, high = 0, self.size + while low < high: + mid = low + high >> 1 + if self.indices[mid] < target: + low = mid + 1 + else: + high = mid + return low + + def insert_child(self, index, child): + pos = self.bisect(index) + self.indices = self.indices[:pos] + index + self.indices[pos:] + self.children.insert(pos, child) + self.size += 1 + + return child + + def get_child(self, index): + for i, char in enumerate(self.indices): + if char == index: + return self.children[i] + + +class RadixTree(object): + def __init__(self): + self.root = RadixTreeNode() + + def __repr__(self): + return repr(self.root) + + def insert(self, key, handler, methods): + i, n, root = 0, len(key), self.root + + def get_position(i): + return n if i == -1 else i + + while i < n: + conflict, num = [], (key[i] == ':') + (root.indices == ':') + + if (root.indices == '*' or + key[i] == '*' and root.indices or + num == 1 or + num == 2 and key[i + 1:get_position( + key.find('/', i))] != root.get_child(':').path): + conflict = [key[:i] + p for p in self.traverse(root)] + + if conflict: + raise Exception('"{}" conflicts with {}'.format(key, conflict)) + + child = root.get_child(key[i]) + + if child is None: + pos = get_position(key.find(':', i)) + if pos == n: + pos = get_position(key.find('*', i)) + if pos == n: + root.insert_child( + key[i], RadixTreeNode(key[i:], handler, methods)) + return + + root = root.insert_child(key[i], RadixTreeNode(key[i:pos])) + root.insert_child( + '*', RadixTreeNode(key[pos + 1:], handler, methods)) + return + + root = root.insert_child(key[i], RadixTreeNode(key[i:pos])) + i = get_position(key.find('/', pos)) + root = root.insert_child(':', RadixTreeNode(key[pos + 1:i])) + + if i == n: + root.add_methods(methods, handler) + else: + root = child + if key[i] == ':': + i += len(root.path) + 1 + if i == n: + root.add_methods(methods, handler) + else: + j, m = 0, len(root.path) + while i < n and j < m and key[i] == root.path[j]: + i += 1 + j += 1 + + if j < m: + child = RadixTreeNode(root.path[j:]) + child.methods = root.methods + child.children = root.children + child.indices = root.indices + child.size = root.size + + root.path = root.path[:j] + root.methods = {} + root.children = [child] + root.indices = child.path[0] + root.size = 1 + + if i == n: + root.add_methods(methods, handler) + + def get(self, key, method): + i, n, root, params = 0, len(key), self.root, {} + while i < n: + if root.indices == ':': + root, pos = root.children[0], key.find('/', i) + if pos == -1: + pos = n + params[root.path], i = key[i:pos], pos + elif root.indices == '*': + root = root.children[0] + params[root.path] = key[i:] + break + else: + root = root.get_child(key[i]) + if root is None: + return False, None, {} + + pos = i + len(root.path) + if key[i:pos] != root.path: + return False, None, {} + i = pos + + return True, root.methods.get(method, None), params + + def traverse(self, root): + r = [] + for i, char in enumerate(root.indices): + child = root.children[i] + path = '{}{}'.format( + char if char in [':', '*'] else '', child.path) + + if child.methods and child.indices: + r.append([path]) + + r.append([path + p for p in self.traverse(child) or ['']]) + return sum(r, []) diff --git a/example/api.py b/example/api.py new file mode 100644 index 0000000..83eb567 --- /dev/null +++ b/example/api.py @@ -0,0 +1,32 @@ +import asyncio + +from loguru import logger + +from busrtworker import App, ConnectionInfo + +app = App() + +api_ci = ConnectionInfo('api', 'localhost:9800', 'busrt.worker.test', static=True, topic='test/#') +caller_ci = ConnectionInfo('caller', 'localhost:9800', 'busrt.worker.test', static=True) +api = app.registry(api_ci) +app.registry(caller_ci) + + +@api.on_call() +def add(a, b): + return a + b + + +@api.subscribe('test/:name') +def print_name(name: str): + logger.info(f'{name} pub message') + +@app.run_on_startup +async def test(server): + async def call(): + await asyncio.sleep(1) + logger.info(f'call remote add result {(await app.caller.add(api_ci.final_name,a=1, b=2))}') + await app.caller.send('test/i_am_caller') + asyncio.create_task(call()) + +app.run() diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..915beb8 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,247 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[package.source] +type = "legacy" +url = "https://mirror.baidu.com/pypi/simple" +reference = "baidu" + +[[package]] +name = "loguru" +version = "0.7.0" +description = "Python logging made (stupidly) simple" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.0-py3-none-any.whl", hash = "sha256:b93aa30099fa6860d4727f1b81f8718e965bb96253fa190fab2077aaad6d15d3"}, + {file = "loguru-0.7.0.tar.gz", hash = "sha256:1612053ced6ae84d7959dd7d5e431a0532642237ec21f7fd83ac73fe539e03e1"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v0.990)", "pre-commit (==3.2.1)", "pytest (==6.1.2)", "pytest (==7.2.1)", "pytest-cov (==2.12.1)", "pytest-cov (==4.0.0)", "pytest-mypy-plugins (==1.10.1)", "pytest-mypy-plugins (==1.9.3)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.2.0)", "tox (==3.27.1)", "tox (==4.4.6)"] + +[package.source] +type = "legacy" +url = "https://mirror.baidu.com/pypi/simple" +reference = "baidu" + +[[package]] +name = "msgpack" +version = "1.0.5" +description = "MessagePack serializer" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a"}, + {file = "msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea"}, + {file = "msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed"}, + {file = "msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c"}, + {file = "msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2"}, + {file = "msgpack-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c"}, + {file = "msgpack-1.0.5-cp36-cp36m-win32.whl", hash = "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9"}, + {file = "msgpack-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a"}, + {file = "msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf"}, + {file = "msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77"}, + {file = "msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0"}, + {file = "msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e"}, + {file = "msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11"}, + {file = "msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc"}, + {file = "msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164"}, + {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, +] + +[package.source] +type = "legacy" +url = "https://mirror.baidu.com/pypi/simple" +reference = "baidu" + +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + +[package.source] +type = "legacy" +url = "https://mirror.baidu.com/pypi/simple" +reference = "baidu" + +[[package]] +name = "zstd" +version = "1.5.5.1" +description = "ZSTD Bindings for Python" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "zstd-1.5.5.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:555779789bc75cd05089c3ba857f45a0a8c4b87d45e5ced02fec77fa8719237a"}, + {file = "zstd-1.5.5.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:86496bd4830cdb7b4b05a9ce6ce2baee87d327ff90845da4ee308452bfbbed4e"}, + {file = "zstd-1.5.5.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:b487c2e67ed42a4e0d47997d209f4456b01b334023083ef61873f79577c84c62"}, + {file = "zstd-1.5.5.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:45ccd45a5b681088fca1a863ca9236ded5112b8011f1d5bf69e908f5eb32023a"}, + {file = "zstd-1.5.5.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:8403fe84207d8b0c7b17bca6c4caad431ac765b1b9b626ad9fae4bb93a64a9d8"}, + {file = "zstd-1.5.5.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:0ab979c6357b8927f0c025ea2f72f25e15d03ce17a8a6c1789e2d5b108bf39ae"}, + {file = "zstd-1.5.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:98cbee6c1b2fe85f02fd475d885f98363c63bc64eebc249d7eb7469a0ff70283"}, + {file = "zstd-1.5.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9962714b89641301029f3832bdf07c20f60b9e64e39e8d7b6253451a82b54f5c"}, + {file = "zstd-1.5.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f59cc92d71537f8082306f75aa403ddb4a4a1069a39f104525673110e4d23f7"}, + {file = "zstd-1.5.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:569f13d0c926ddafceebce8ac73baddfc2bd9cbbbbc922b6b3073338cc43dae6"}, + {file = "zstd-1.5.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ba530c44f252016acc6ef906d7d2070c1ad0cfe835c498fdcd37493e4772ac6e"}, + {file = "zstd-1.5.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8ee3496ed8fff3add6c6e658b207f18d96474c3db0c28ab7a69623380b1a0a8c"}, + {file = "zstd-1.5.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:530d69bea2791cde8afa7fe988f3a37c3ba37015f6a1d5593c0500f089f3090e"}, + {file = "zstd-1.5.5.1-cp310-cp310-win32.whl", hash = "sha256:cf179e51f447b6a7ff47e449fcb98fb5fe15aedcc90401697cf7c93dd6e4434e"}, + {file = "zstd-1.5.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:5f5e6e0805d710d7509c8d175a467eb89c631a4142b1a630ceeb8e3e3138d152"}, + {file = "zstd-1.5.5.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:022f935a8666e08f0fff6204938a84d9fe4fcd8235a205787275933a07a164fb"}, + {file = "zstd-1.5.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3d15a2d18dac8bcafdde52fdf5d40ecae1f73b7de19b171f42339d2e51346d0"}, + {file = "zstd-1.5.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45b9c67989f50ba63ffa0c50c9eaa037c2d14abacb0813e838ad705135245b4b"}, + {file = "zstd-1.5.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97da6a842ba7e4acf8bba7c596057143ee39b3c4a467196c2096d460e44accd6"}, + {file = "zstd-1.5.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dafd492fb8ee4ae04c81ab00f5f137860e7071f611335dd4cdb1c38bd8f11bc"}, + {file = "zstd-1.5.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9ee83e0bcbfd776200b026b3b9e86c6c86b8f414749f58d87c85dcf456b27066"}, + {file = "zstd-1.5.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ae2fd4bc8ea772a7b5f1acd1cac9e34bb9cd8fcde191f170092fdeea779a3a12"}, + {file = "zstd-1.5.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:edea52a0109f48fd46f4763689d3d356dcafd20ddf6789c559a1bd2e62b40a32"}, + {file = "zstd-1.5.5.1-cp311-cp311-win32.whl", hash = "sha256:88410481209520298ec4430e0d1d57e004c45e0b27c3035674fb182ccd2d8b7b"}, + {file = "zstd-1.5.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:dce18aaefbacf8b133367be86beec670baf68c0420bfcca49be08dbdbf933db6"}, + {file = "zstd-1.5.5.1-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:634dc632f7cf87e95dabf74dcf682e3507bd5cb9dd1bcdb81f92a6521aab0bd2"}, + {file = "zstd-1.5.5.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:608414eb75ead573891d97a1e529848b8f31749d21a440e80838548a19d8c0e6"}, + {file = "zstd-1.5.5.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:384128f7a731e3f45da49976591cec03fc4079e70653df10d9ea43a1d3b49d50"}, + {file = "zstd-1.5.5.1-cp35-cp35m-win32.whl", hash = "sha256:4bce254174ef05cea01021d67e18489d5d08db1168e758b62ecee121572a52a9"}, + {file = "zstd-1.5.5.1-cp35-cp35m-win_amd64.whl", hash = "sha256:3f0ff81232b49d7eb4f4d9e6f92443c9d242c139ad98ffedac0e889568f900ce"}, + {file = "zstd-1.5.5.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:a871df41b801a260cc849c2c76f300ebb9d286c4b7a1fd6ce45fe0c91340b767"}, + {file = "zstd-1.5.5.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5a53860dbfbea281eb690ce09cae28967cf1df8e6d7560e4a8bf5b9fcb258147"}, + {file = "zstd-1.5.5.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:a37cbc0580fdfd66c8b3ec65f9af00a4a34e9781b54dfb89f04d301dc375c90a"}, + {file = "zstd-1.5.5.1-cp36-cp36m-win32.whl", hash = "sha256:5531b683539ae1f7b2ad23dacee8a73e5d7eaa6702ea8df5a24bd3318647dee1"}, + {file = "zstd-1.5.5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eeaff418269b41eee8c7971fbba9d32d07d3f6aa26f962a72aff725071096a1b"}, + {file = "zstd-1.5.5.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:8bd6a9050de8bbe844447348372ca17d01bc05207619f6a5d448567d111b5cd9"}, + {file = "zstd-1.5.5.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2ece3d20ef357370584f304407fbd1e4ff9c231209320e08a889b8e3725d56e"}, + {file = "zstd-1.5.5.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:687f9e03dc9f9b8803840425bb23bf6bc700888b4860afcf43c4f238102752d2"}, + {file = "zstd-1.5.5.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a649daac9c8f1b37d29f2b3d0a43f134061659b54877fe4b0da6df2965dc91f"}, + {file = "zstd-1.5.5.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:bddc7e3c3ce31c01fe1edaa7c03c0b9e71eadf4ce1609746d32f86d95a0449e6"}, + {file = "zstd-1.5.5.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:12bf8e04add8bb84f9fe9117f3de6d9394eade6a5a82fe4d6bd95914fc6ef423"}, + {file = "zstd-1.5.5.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9e6a15fa4d2e65c5902ab2a4e41279ac126cb371ce6c3c75ad5789bb20dd1f54"}, + {file = "zstd-1.5.5.1-cp37-cp37m-win32.whl", hash = "sha256:a1c269243a4321beb948635b544ccbe6390846358ace620fd000ab7099011d9c"}, + {file = "zstd-1.5.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:91366e36773241cb4b049a32f4495d33dd274df1eea5b55396f5f3984a3de22e"}, + {file = "zstd-1.5.5.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:d3ce2cb310690994274d133ea7f269dd4b81799fdbce158690556209723d7d4e"}, + {file = "zstd-1.5.5.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e0c87bfbfa9d852f79c90bcd7426c3ba46cf3285e6984013636d4fc854ba9230"}, + {file = "zstd-1.5.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce6d829d515f272fddb3a87e1a5f32cc0f1a7b0cba24d360c89f4a165b74b"}, + {file = "zstd-1.5.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e05f81f346213b23ed1b12d84fc1f72e65eacd8978e1e88facf185c82bd3d053"}, + {file = "zstd-1.5.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43ec66c4c3a76351c672c6ef9f0ff3412fca9ede0a56d18dddaf6418a93faef8"}, + {file = "zstd-1.5.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:58e554e91e0d49f4f2b2df390cdd0f64aa9b6fd5f4dcb208c094bfd079b30f3a"}, + {file = "zstd-1.5.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:883c6d3b6f5574e1765ca97f4b6a41b69094a41be56175552faebc0e0e43b65e"}, + {file = "zstd-1.5.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d52b6932cab5419c434bccfea3e5640e755369fc9eeb51e3d17e15bf8e8cb103"}, + {file = "zstd-1.5.5.1-cp38-cp38-win32.whl", hash = "sha256:dcaf44270ec88552e969be4dd3359b34aa3065663ccd8168a257c78f150a356c"}, + {file = "zstd-1.5.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:627f12cb7035723c8f3d8d4cefcad6d950ed9cba33fd3eb46bae04ccab479234"}, + {file = "zstd-1.5.5.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:c0dab132c1a5a7cc838a7c3e4e380ad153b9d7bd1fadafabf6cfeb780b916201"}, + {file = "zstd-1.5.5.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d4ab0a5dd9a41d3b083304beee7ada40ee36431acbeb75132032f4fe5cf0490a"}, + {file = "zstd-1.5.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f6e38f496d287020658c6b4cdb5e815ecc6998889bd0f1f9ab0825f2e3d74ef"}, + {file = "zstd-1.5.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0096c8ee0ed4bfe406bc961019f55552109e19771bfd3eb32d2af56ea27085c"}, + {file = "zstd-1.5.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a0f1527728c50b6aa8f04b47a07580f0ae13cfc6c6d9c96bb0bdf5259487559"}, + {file = "zstd-1.5.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6a64e420c904063c5c3de53c00ec0993ebc0a48cebbef97dc6c768562c5abab5"}, + {file = "zstd-1.5.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03444e357b7632c64480a81ce7095242dab9d7f8aed317326563ef6c663263eb"}, + {file = "zstd-1.5.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:88b9a10f80d2b87bf8cc1a1fc20a815ed92b5eefdc15cbe8062021f0b5a26a10"}, + {file = "zstd-1.5.5.1-cp39-cp39-win32.whl", hash = "sha256:c91cc1606eb8b3a6fed11faaef4c6e55f1133d70cf0db0c829a2cf9c2ac1dfd9"}, + {file = "zstd-1.5.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:f462e2ebf26dcbfc2c8dddd6b5c56859683f0b77edb8f268e637f7d390a58f74"}, + {file = "zstd-1.5.5.1-pp27-pypy_73-macosx_10_14_x86_64.whl", hash = "sha256:c63f916732e3e309e49ec95e7a0af5d37ff1321f3df2aac10e507bd2b56fceda"}, + {file = "zstd-1.5.5.1-pp27-pypy_73-manylinux1_x86_64.whl", hash = "sha256:50d4850d758bb033df50722cc13ed913b2afcd5385250be4f3ffb79a26b319c3"}, + {file = "zstd-1.5.5.1-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:0412d666515e78a91ada7e2d78e9dd6b25ddda1b41623b145b99653275c7f3ce"}, + {file = "zstd-1.5.5.1-pp36-pypy36_pp73-macosx_10_14_x86_64.whl", hash = "sha256:0ea91f74869a3cdcb2dde08f8f30ee3da72782c5d1737afed9c703232815864e"}, + {file = "zstd-1.5.5.1-pp36-pypy36_pp73-manylinux1_x86_64.whl", hash = "sha256:477548897dc2b8b595af7bec5f0f55dcba8e9a282335f687cc663b52b171357b"}, + {file = "zstd-1.5.5.1-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:c518938b57a56001ee04dcf79a432152f5bd431416f3b22819ba959bc6054d89"}, + {file = "zstd-1.5.5.1-pp36-pypy36_pp73-win32.whl", hash = "sha256:894a8fe0228d5e24dc286a8d98eb0ce2883f8e2e57f3b7e7619ebdb67967120a"}, + {file = "zstd-1.5.5.1-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:42ec0a4ae9bedd9909fa4f580f3c800469da1b631faeaa94f204e1b66c767fa2"}, + {file = "zstd-1.5.5.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d56dedaa04ab8ecc23492972b12e0bf8529f64c9bceb28c11f43c2369c9768b3"}, + {file = "zstd-1.5.5.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5b060770d796e4c01f5848b345c3cea8a177ab4e7cd95a1963a355042d429e1"}, + {file = "zstd-1.5.5.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fea04805ef6e1cb93d6e5d6bbc7a03bc75a5c733fd352d5aaa81109986fdf1ef"}, + {file = "zstd-1.5.5.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:405c28a35756e57a434bbd7ed29dc5e6490cd2fc2118cbf78b60eaebd134f5e9"}, + {file = "zstd-1.5.5.1-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:c42e630443b01a891277426365a51a2aa630b059ce675992c70c1928d30eccb4"}, + {file = "zstd-1.5.5.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1520d23f24f26cdfbcdb4dc86947446b8f694838bfce728d7fc4b3492397357c"}, + {file = "zstd-1.5.5.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4730737f63cf802321743ded6acc85e747e7f5587c5ba2e51a760bf009f7de"}, + {file = "zstd-1.5.5.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9f8c014395e89ad7f67ffe873c0fa1d8e9b4dea8b1801d24e8d9ccd8259858d"}, + {file = "zstd-1.5.5.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5d9ba4f6af0945809bfa3387c6a1208a22937a876521b9ec347e7183d623311b"}, + {file = "zstd-1.5.5.1-pp39-pypy39_pp73-macosx_10_14_x86_64.whl", hash = "sha256:04dfd9f46b0b0b1bc413884fe028b726febcb726d4f66e3cf8afc00c2d9026bf"}, + {file = "zstd-1.5.5.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af52436a2eb5caa925d95461973984cb34d472a963b6be1c0a9f2dfbafad096f"}, + {file = "zstd-1.5.5.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610928b888a2e7ae9d2018ffa814859d47ec4ba75f89a1188ab4eb9232636ee5"}, + {file = "zstd-1.5.5.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee3c9feea99c7f4ff43129a885da056b5aa0cde3f7876bf6397bfb9433f44352"}, + {file = "zstd-1.5.5.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6ac9768eeb3c6b530db93de2fec9b363776075dc8a00ee4049612ba5397ca8e"}, + {file = "zstd-1.5.5.1.tar.gz", hash = "sha256:1ef980abf0e1e072b028d2d76ef95b476632651c96225cf30b619c6eef625672"}, +] + +[package.source] +type = "legacy" +url = "https://mirror.baidu.com/pypi/simple" +reference = "baidu" + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "04b38a8ef5153eb24a8822947e6897b5f9c2459eeda498ca54f243cfedb196f5" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3fe637b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "busrt-worker" +version = "0.1.0" +description = "busrt-worker is a Python-based async busrt message handle framework" +authors = ["JimZhang "] +readme = "README.md" +packages = [{include = "busrt_worker"}] + +[tool.poetry.dependencies] +python = "^3.10" +msgpack = "^1.0.5" +zstd = "^1.5.5.1" +loguru = "^0.7.0" + + +[tool.poetry.group.dev.dependencies] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + + +[[tool.poetry.source]] +name = "baidu" +url = "https://mirror.baidu.com/pypi/simple" +default = true + + + +[tool.ruff] +exclude = [ + "tests", +] +line-length = 120 +select = [ + "E", + "F", + "W", + "I" +] +target-version = "py310" +[tool.ruff.isort] +combine-as-imports = true \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29