From d861b2fe45b9cbae0c8415a406504861b31d8571 Mon Sep 17 00:00:00 2001 From: hkc Date: Tue, 1 Nov 2022 12:55:23 +0300 Subject: [PATCH 1/7] Uhh I think I added logging and some other stuff Yeah so basically now initialization of integrations and filters is moved to from_section instead of __init__ because muh better imo --- README.md | 14 ++++----- mastoposter/__init__.py | 22 ++++++++++++-- mastoposter/__main__.py | 32 ++++++++++++++------ mastoposter/filters/__init__.py | 21 ++++++++++++- mastoposter/filters/base.py | 10 ++++-- mastoposter/filters/boost.py | 11 +++++-- mastoposter/filters/combined.py | 15 ++++++--- mastoposter/filters/media.py | 24 ++++++++++----- mastoposter/filters/mention.py | 21 ++++++++----- mastoposter/filters/spoiler.py | 10 ++++-- mastoposter/filters/text.py | 29 +++++++++++------- mastoposter/filters/visibility.py | 11 +++++-- mastoposter/integrations/base.py | 7 ++++- mastoposter/integrations/discord/__init__.py | 8 +++-- mastoposter/integrations/telegram.py | 31 +++++++++++++++---- mastoposter/utils.py | 14 +++++++++ 16 files changed, 209 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 4c77cb3..130ff1e 100644 --- a/README.md +++ b/README.md @@ -175,15 +175,13 @@ expression to match your spoiler text. Filter to match post content against either a regular expression, or a list of tags. Matching is done on the plaintext version of the post. -`mode` property determines the type of operation. Can be either `regexp`, -`tag` or `hashtag` (two last ones are the same). +You can have one of two properties (but not both because fuck you): `tags` or +`regexp`. It's obvious what does what: if you have `regexp` set, plaintext +version of status is checked against that regular expression, and if you have +`tags` set, then only statuses that have those tags will be allowed. -In `mode = regexp` you have to specify the `regexp` option with a regular -expression to match against. - -In `mode = tag` or `mode = hashtag`, `tags` option should contain a -space-separated list of tags. If any of the tags are present in that list, -filter will be triggered. +Please note that in case of tags, you should NOT use `#` symbol in front of +them. #### `type = visibility` Simple filter that just checks for post visibility. diff --git a/mastoposter/__init__.py b/mastoposter/__init__.py index a096cb5..1c4c717 100644 --- a/mastoposter/__init__.py +++ b/mastoposter/__init__.py @@ -1,5 +1,6 @@ from asyncio import gather from configparser import ConfigParser +from logging import getLogger from typing import Dict, List, Optional from mastoposter.filters import run_filters from mastoposter.filters.base import BaseFilter, FilterInstance @@ -11,33 +12,49 @@ from mastoposter.integrations import ( ) from mastoposter.types import Status +logger = getLogger(__name__) + def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]: modules: List[FilteredIntegration] = [] for module_name in config.get("main", "modules").split(): mod = config[f"module/{module_name}"] + logger.info( + "Configuring %s integration (type=%s)", module_name, mod["type"] + ) filters: Dict[str, FilterInstance] = {} for filter_name in mod.get("filters", "").split(): filter_basename = filter_name.lstrip("~!") + logger.info( + "Loading filter %s for integration %s", + filter_basename, + module_name, + ) + filters[filter_basename] = BaseFilter.new_instance( filter_name, config[f"filter/{filter_basename}"] ) for finst in list(filters.values()): + logger.info("Running post-initialization hook for %r", finst) finst.filter.post_init(filters, config) + # TODO: make a registry of integrations + # INFO: mastoposter/integrations/base.py@__init__ if mod["type"] == "telegram": modules.append( FilteredIntegration( - TelegramIntegration(mod), list(filters.values()) + TelegramIntegration.from_section(mod), + list(filters.values()), ) ) elif mod["type"] == "discord": modules.append( FilteredIntegration( - DiscordIntegration(mod), list(filters.values()) + DiscordIntegration.from_section(mod), + list(filters.values()), ) ) else: @@ -48,6 +65,7 @@ def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]: async def execute_integrations( status: Status, sinks: List[FilteredIntegration] ) -> List[Optional[str]]: + logger.debug("Executing integrations...") return await gather( *[sink[0](status) for sink in sinks if run_filters(sink[1], status)], return_exceptions=True, diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index e95aa8f..083588d 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from asyncio import run from configparser import ConfigParser, ExtendedInterpolation +from logging import getLogger from mastoposter import execute_integrations, load_integrations_from from mastoposter.integrations import FilteredIntegration from mastoposter.sources import websocket_source @@ -8,10 +9,14 @@ from typing import AsyncGenerator, Callable, List from mastoposter.types import Account, Status from httpx import Client +from mastoposter.utils import normalize_config + WSOCK_TEMPLATE = "wss://{instance}/api/v1/streaming" VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials" +logger = getLogger() + async def listen( source: Callable[..., AsyncGenerator[Status, None]], @@ -20,12 +25,19 @@ async def listen( /, **kwargs, ): + logger.info("Starting listening...") async for status in source(**kwargs): + logger.debug("Got status: %r", status) if status.account.id != user: continue # TODO: add option/filter to handle that if status.visibility in ("direct",): + logger.info( + "Skipping post %s (status.visibility=%r)", + status.uri, + status.visibility, + ) continue # TODO: find a better way to handle threads @@ -33,6 +45,10 @@ async def listen( status.in_reply_to_account_id is not None and status.in_reply_to_account_id != user ): + logger.info( + "Skipping post %s because it's a reply to another person", + status.uri, + ) continue await execute_integrations(status, drains) @@ -42,21 +58,15 @@ def main(config_path: str): conf = ConfigParser(interpolation=ExtendedInterpolation()) conf.read(config_path) - for section in conf.sections(): - _remove = set() - for k, v in conf[section].items(): - normalized_key = k.replace(" ", "_").replace("-", "_") - if k == normalized_key: - continue - conf[section][normalized_key] = v - _remove.add(k) - for k in _remove: - del conf[section][k] + normalize_config(conf) modules: List[FilteredIntegration] = load_integrations_from(conf) + logger.info("Loaded %d integrations", len(modules)) + user_id: str = conf["main"]["user"] if user_id == "auto": + logger.info("config.main.user is set to auto, getting user ID") with Client() as c: rq = c.get( VERIFY_CREDS_TEMPLATE.format(**conf["main"]), @@ -64,8 +74,10 @@ def main(config_path: str): ) account = Account.from_dict(rq.json()) user_id = account.id + logger.info("account.id=%s", user_id) url = "wss://{}/api/v1/streaming".format(conf["main"]["instance"]) + run( listen( websocket_source, diff --git a/mastoposter/filters/__init__.py b/mastoposter/filters/__init__.py index 45d14d0..21398bd 100644 --- a/mastoposter/filters/__init__.py +++ b/mastoposter/filters/__init__.py @@ -1,3 +1,4 @@ +from logging import getLogger from typing import List from mastoposter.types import Status @@ -10,8 +11,26 @@ from mastoposter.filters.text import TextFilter # NOQA from mastoposter.filters.spoiler import SpoilerFilter # NOQA from mastoposter.filters.visibility import VisibilityFilter # NOQA +logger = getLogger(__name__) + def run_filters(filters: List[FilterInstance], status: Status) -> bool: + logger.debug("Running filters on %r", status) + if not filters: + logger.debug("No filters, returning True") return True - return all((fil.filter(status) ^ fil.inverse for fil in filters)) + + results: List[bool] = [] + for fil in filters: + result = fil.filter(status) + logger.debug( + "%r -> %r ^ %r -> %r", + fil.filter, + result, + fil.inverse, + result ^ fil.inverse, + ) + results.append(result ^ fil.inverse) + logger.debug("Result: %r", all(results)) + return all(results) diff --git a/mastoposter/filters/base.py b/mastoposter/filters/base.py index 9513790..350d844 100644 --- a/mastoposter/filters/base.py +++ b/mastoposter/filters/base.py @@ -23,8 +23,8 @@ class BaseFilter(ABC): filter_name: ClassVar[str] = "_base" - def __init__(self, section: SectionProxy): - UNUSED(section) + def __init__(self): + pass def __init_subclass__(cls, filter_name: str, **kwargs): super().__init_subclass__(**kwargs) @@ -51,7 +51,11 @@ class BaseFilter(ABC): def load_filter(cls, name: str, section: SectionProxy) -> "BaseFilter": if name not in cls.FILTER_REGISTRY: raise KeyError(f"no filter with name {name!r} was found") - return cls.FILTER_REGISTRY[name](section) + return cls.FILTER_REGISTRY[name].from_section(section) + + @classmethod + def from_section(cls, section: SectionProxy) -> "BaseFilter": + raise NotImplementedError @classmethod def new_instance(cls, name: str, section: SectionProxy) -> FilterInstance: diff --git a/mastoposter/filters/boost.py b/mastoposter/filters/boost.py index 687b60b..448e2b3 100644 --- a/mastoposter/filters/boost.py +++ b/mastoposter/filters/boost.py @@ -1,13 +1,18 @@ from configparser import SectionProxy from fnmatch import fnmatch +from typing import List from mastoposter.filters.base import BaseFilter from mastoposter.types import Status class BoostFilter(BaseFilter, filter_name="boost"): - def __init__(self, section: SectionProxy): - super().__init__(section) - self.list = section.get("list", "").split() + def __init__(self, accounts: List[str]): + super().__init__() + self.list = accounts + + @classmethod + def from_section(cls, section: SectionProxy) -> "BoostFilter": + return cls(section["list"].split()) @classmethod def check_account(cls, acct: str, mask: str) -> bool: diff --git a/mastoposter/filters/combined.py b/mastoposter/filters/combined.py index 7bcf899..0700cd4 100644 --- a/mastoposter/filters/combined.py +++ b/mastoposter/filters/combined.py @@ -11,12 +11,19 @@ class CombinedFilter(BaseFilter, filter_name="combined"): "single": lambda d: sum(d) == 1, } - def __init__(self, section: SectionProxy): - self.filter_names = section.get("filters", "").split() - self.operator = self.OPERATORS[section.get("operator", "all")] - self._operator_name = section.get("operator", "all") + def __init__(self, filter_names: List[str], operator: str): + self.filter_names = filter_names + self._operator_name = operator + self.operator = self.OPERATORS[self._operator_name] self.filters: List[FilterInstance] = [] + @classmethod + def from_section(cls, section: SectionProxy) -> "CombinedFilter": + return cls( + filter_names=section["filters"].split(), + operator=section.get("operator", "all"), + ) + def post_init( self, filters: Dict[str, FilterInstance], config: ConfigParser ): diff --git a/mastoposter/filters/media.py b/mastoposter/filters/media.py index 39c6a39..9f01ba9 100644 --- a/mastoposter/filters/media.py +++ b/mastoposter/filters/media.py @@ -1,16 +1,26 @@ from configparser import SectionProxy -from typing import Set +from typing import Literal, Set from mastoposter.filters.base import BaseFilter from mastoposter.types import Status class MediaFilter(BaseFilter, filter_name="media"): - def __init__(self, section: SectionProxy): - super().__init__(section) - self.valid_media: Set[str] = set(section.get("valid_media").split()) - self.mode = section.get("mode", "include") - if self.mode not in ("include", "exclude", "only"): - raise ValueError(f"{self.mode=} is not valid") + def __init__( + self, + valid_media: Set[str], + mode: Literal["include", "exclude", "only"], + ): + super().__init__() + self.valid_media: Set[str] = valid_media + self.mode = mode + assert self.mode in ("include", "exclude", "only") + + @classmethod + def from_section(cls, section: SectionProxy) -> "MediaFilter": + return cls( + valid_media=set(section["valid_media"].split()), + mode=section.get("mode", "include"), # type: ignore + ) def __call__(self, status: Status) -> bool: if not status.media_attachments: diff --git a/mastoposter/filters/mention.py b/mastoposter/filters/mention.py index 8ba70e4..4046d95 100644 --- a/mastoposter/filters/mention.py +++ b/mastoposter/filters/mention.py @@ -1,6 +1,6 @@ from configparser import SectionProxy from re import Pattern, compile as regexp -from typing import ClassVar +from typing import ClassVar, Set from fnmatch import fnmatch from mastoposter.filters.base import BaseFilter from mastoposter.types import Status @@ -9,22 +9,27 @@ from mastoposter.types import Status class MentionFilter(BaseFilter, filter_name="mention"): MENTION_REGEX: ClassVar[Pattern] = regexp(r"@([^@]+)(@([^@]+))?") - def __init__(self, section: SectionProxy): - super().__init__(section) - self.list = section.get("list", "").split() + def __init__(self, accounts: Set[str]): + super().__init__() + self.accounts = accounts + + @classmethod + def from_section(cls, section: SectionProxy) -> "MentionFilter": + return cls(set(section.get("list", "").split())) @classmethod def check_account(cls, acct: str, mask: str) -> bool: return fnmatch("@" + acct, mask) def __call__(self, status: Status) -> bool: - if not self.list and status.mentions: + if not self.accounts and status.mentions: return True + # XXX: make it better somehow. and faster. and stronger. return any( ( any( self.check_account(mention.acct, mask) - for mask in self.list + for mask in self.accounts ) for mention in status.mentions ) @@ -32,5 +37,7 @@ class MentionFilter(BaseFilter, filter_name="mention"): def __repr__(self): return str.format( - "Filter:{name}({list!r})", name=self.filter_name, list=self.list + "Filter:{name}({list!r})", + name=self.filter_name, + list=self.accounts, ) diff --git a/mastoposter/filters/spoiler.py b/mastoposter/filters/spoiler.py index 6e23fcd..0d862e0 100644 --- a/mastoposter/filters/spoiler.py +++ b/mastoposter/filters/spoiler.py @@ -5,9 +5,13 @@ from mastoposter.types import Status class SpoilerFilter(BaseFilter, filter_name="spoiler"): - def __init__(self, section: SectionProxy): - super().__init__(section) - self.regexp: Pattern = regexp(section.get("regexp", "^.*$")) + def __init__(self, regex: str = "^.*$"): + super().__init__() + self.regexp: Pattern = regexp(regex) + + @classmethod + def from_section(cls, section: SectionProxy) -> "SpoilerFilter": + return cls(section.get("regexp", section.get("regex", "^.*$"))) def __call__(self, status: Status) -> bool: return self.regexp.match(status.spoiler_text) is not None diff --git a/mastoposter/filters/text.py b/mastoposter/filters/text.py index eb7738f..56bc28d 100644 --- a/mastoposter/filters/text.py +++ b/mastoposter/filters/text.py @@ -8,18 +8,24 @@ from mastoposter.types import Status class TextFilter(BaseFilter, filter_name="content"): - def __init__(self, section: SectionProxy): - super().__init__(section) - self.mode = section["mode"] - self.tags: Set[str] = set() - self.regexp: Optional[Pattern] = None + def __init__( + self, regex: Optional[str] = None, tags: Optional[Set[str]] = None + ): + super().__init__() + assert regex is not None or tags - if self.mode == "regexp": - self.regexp = regexp(section["regexp"]) - elif self.mode in ("hashtag", "tag"): - self.tags = set(map(str.lower, section["tags"].split())) - else: - raise ValueError(f"Invalid filter mode {self.mode}") + self.tags: Optional[Set[str]] = tags + self.regexp: Optional[Pattern] = regexp(regex) if regex else None + + @classmethod + def from_section(cls, section: SectionProxy) -> "TextFilter": + if "regexp" in section and "tags" in section: + raise AssertionError("you can't use both tags and regexp") + elif "regexp" in section: + return cls(regex=section["regexp"]) + elif "tags" in section: + return cls(tags=set(section["tags"].split())) + raise AssertionError("neither regexp or tags were set") @classmethod def node_to_text(cls, el: PageElement) -> str: @@ -63,3 +69,4 @@ class TextFilter(BaseFilter, filter_name="content"): name=self.filter_name, tags=self.tags, ) + return "Filter:{name}(invalid)".format(name=self.filter_name) diff --git a/mastoposter/filters/visibility.py b/mastoposter/filters/visibility.py index 70fda56..40b3b8a 100644 --- a/mastoposter/filters/visibility.py +++ b/mastoposter/filters/visibility.py @@ -1,12 +1,17 @@ from configparser import SectionProxy +from typing import Set from mastoposter.filters.base import BaseFilter from mastoposter.types import Status class VisibilityFilter(BaseFilter, filter_name="visibility"): - def __init__(self, section: SectionProxy): - super().__init__(section) - self.options = set(section["options"].split()) + def __init__(self, options: Set[str]): + super().__init__() + self.options = options + + @classmethod + def from_section(cls, section: SectionProxy) -> "BaseFilter": + return cls(set(section["options"].split())) def __call__(self, status: Status) -> bool: return status.visibility in self.options diff --git a/mastoposter/integrations/base.py b/mastoposter/integrations/base.py index 1b6765c..815263a 100644 --- a/mastoposter/integrations/base.py +++ b/mastoposter/integrations/base.py @@ -6,9 +6,14 @@ from mastoposter.types import Status class BaseIntegration(ABC): - def __init__(self, section: SectionProxy): + # TODO: make a registry of integrations + def __init__(self): pass + @classmethod + def from_section(cls, section: SectionProxy) -> "BaseIntegration": + raise NotImplementedError + @abstractmethod async def __call__(self, status: Status) -> Optional[str]: raise NotImplementedError diff --git a/mastoposter/integrations/discord/__init__.py b/mastoposter/integrations/discord/__init__.py index 38b96bb..417c9dd 100644 --- a/mastoposter/integrations/discord/__init__.py +++ b/mastoposter/integrations/discord/__init__.py @@ -12,8 +12,12 @@ from mastoposter.types import Status class DiscordIntegration(BaseIntegration): - def __init__(self, section: SectionProxy): - self.webhook = section.get("webhook", "") + def __init__(self, webhook: str): + self.webhook = webhook + + @classmethod + def from_section(cls, section: SectionProxy) -> "DiscordIntegration": + return cls(section["webhook"]) async def execute_webhook( self, diff --git a/mastoposter/integrations/telegram.py b/mastoposter/integrations/telegram.py index a67bfde..fb81b07 100644 --- a/mastoposter/integrations/telegram.py +++ b/mastoposter/integrations/telegram.py @@ -54,12 +54,31 @@ Boost from \ class TelegramIntegration(BaseIntegration): - def __init__(self, sect: SectionProxy): - self.token = sect.get("token", "") - self.chat_id = sect.get("chat", "") - self.silent = sect.getboolean("silent", True) - self.template: Template = Template( - emojize(sect.get("template", DEFAULT_TEMPLATE)) + def __init__( + self, + token: str, + chat_id: str, + template: Optional[Template] = None, + silent: bool = True, + ): + self.token = token + self.chat_id = chat_id + self.silent = silent + + if template is None: + self.template = Template(emojize(DEFAULT_TEMPLATE)) + else: + self.template = template + + @classmethod + def from_section(cls, section: SectionProxy) -> "TelegramIntegration": + return cls( + token=section["token"], + chat_id=section["chat"], + silent=section.getboolean("silent", True), + template=Template( + emojize(section.get("template", DEFAULT_TEMPLATE)) + ), ) async def _tg_request(self, method: str, **kwargs) -> TGResponse: diff --git a/mastoposter/utils.py b/mastoposter/utils.py index 30e0352..14e6bcc 100644 --- a/mastoposter/utils.py +++ b/mastoposter/utils.py @@ -1,3 +1,4 @@ +from configparser import ConfigParser from html import escape from typing import Callable, Dict from bs4.element import Tag, PageElement @@ -16,6 +17,19 @@ def md_escape(text: str) -> str: ) +def normalize_config(conf: ConfigParser): + for section in conf.sections(): + _remove = set() + for k, v in conf[section].items(): + normalized_key = k.replace(" ", "_").replace("-", "_") + if k == normalized_key: + continue + conf[section][normalized_key] = v + _remove.add(k) + for k in _remove: + del conf[section][k] + + def node_to_html(el: PageElement) -> str: TAG_TRANSFORMS: Dict[str, Callable[[Tag,], str]] = { "a": lambda tag: '{}'.format( From 1ec78d5f53bdaa2fb07d3e2cc3cacb98d713495b Mon Sep 17 00:00:00 2001 From: hkc Date: Tue, 1 Nov 2022 13:04:31 +0300 Subject: [PATCH 2/7] Forgot to initialize logger --- mastoposter/__main__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index 083588d..4073ab0 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 from asyncio import run from configparser import ConfigParser, ExtendedInterpolation -from logging import getLogger +from logging import DEBUG, Formatter, StreamHandler, getLogger +from sys import stdout from mastoposter import execute_integrations, load_integrations_from from mastoposter.integrations import FilteredIntegration from mastoposter.sources import websocket_source @@ -18,6 +19,14 @@ VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials" logger = getLogger() +def init_logger(): + stdout_handler = StreamHandler(stdout) + stdout_handler.setLevel(DEBUG) + formatter = Formatter("[%(asctime)s][%(levelname)5s:%(name)s] %(message)s") + stdout_handler.setFormatter(formatter) + logger.addHandler(stdout_handler) + + async def listen( source: Callable[..., AsyncGenerator[Status, None]], drains: List[FilteredIntegration], @@ -55,6 +64,7 @@ async def listen( def main(config_path: str): + init_logger() conf = ConfigParser(interpolation=ExtendedInterpolation()) conf.read(config_path) From 95c9a523e019629f59847f8f6a0c44de5d24ad50 Mon Sep 17 00:00:00 2001 From: hkc Date: Tue, 1 Nov 2022 13:37:47 +0300 Subject: [PATCH 3/7] More logging? --- mastoposter/__main__.py | 3 ++- mastoposter/filters/__init__.py | 2 +- mastoposter/integrations/discord/__init__.py | 6 ++++++ mastoposter/integrations/telegram.py | 20 +++++++++++++++++++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index 4073ab0..96ebfc8 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -19,12 +19,13 @@ VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials" logger = getLogger() -def init_logger(): +def init_logger(loglevel: int = DEBUG): stdout_handler = StreamHandler(stdout) stdout_handler.setLevel(DEBUG) formatter = Formatter("[%(asctime)s][%(levelname)5s:%(name)s] %(message)s") stdout_handler.setFormatter(formatter) logger.addHandler(stdout_handler) + logger.setLevel(loglevel) async def listen( diff --git a/mastoposter/filters/__init__.py b/mastoposter/filters/__init__.py index 21398bd..015245c 100644 --- a/mastoposter/filters/__init__.py +++ b/mastoposter/filters/__init__.py @@ -15,7 +15,7 @@ logger = getLogger(__name__) def run_filters(filters: List[FilterInstance], status: Status) -> bool: - logger.debug("Running filters on %r", status) + logger.debug("Running filters on %r", status.id) if not filters: logger.debug("No filters, returning True") diff --git a/mastoposter/integrations/discord/__init__.py b/mastoposter/integrations/discord/__init__.py index 417c9dd..bc674d4 100644 --- a/mastoposter/integrations/discord/__init__.py +++ b/mastoposter/integrations/discord/__init__.py @@ -1,4 +1,5 @@ from configparser import SectionProxy +from logging import getLogger from typing import List, Optional from httpx import AsyncClient from zlib import crc32 @@ -10,6 +11,8 @@ from mastoposter.integrations.discord.types import ( ) from mastoposter.types import Status +logger = getLogger(__name__) + class DiscordIntegration(BaseIntegration): def __init__(self, webhook: str): @@ -35,6 +38,9 @@ class DiscordIntegration(BaseIntegration): if embeds is not None else [], } + + logger.debug("Executing webhook with %r", json) + return ( await c.post( self.webhook, diff --git a/mastoposter/integrations/telegram.py b/mastoposter/integrations/telegram.py index fb81b07..4108dd3 100644 --- a/mastoposter/integrations/telegram.py +++ b/mastoposter/integrations/telegram.py @@ -1,5 +1,6 @@ from configparser import SectionProxy from dataclasses import dataclass +from logging import getLogger from typing import Any, List, Mapping, Optional from httpx import AsyncClient from jinja2 import Template @@ -8,6 +9,9 @@ from mastoposter.types import Attachment, Poll, Status from emoji import emojize +logger = getLogger(__name__) + + @dataclass class TGResponse: ok: bool @@ -89,6 +93,7 @@ class TelegramIntegration(BaseIntegration): ) async def _post_plaintext(self, text: str) -> TGResponse: + logger.debug("Sending HTML message: %r", text) return await self._tg_request( "sendMessage", parse_mode="HTML", @@ -101,6 +106,9 @@ class TelegramIntegration(BaseIntegration): async def _post_media(self, text: str, media: Attachment) -> TGResponse: # Just to be safe if media.type not in MEDIA_MAPPING: + logger.warning( + "Media %r has unknown type, falling back to plaintext", media + ) return await self._post_plaintext(text) return await self._tg_request( @@ -116,6 +124,7 @@ class TelegramIntegration(BaseIntegration): async def _post_mediagroup( self, text: str, media: List[Attachment] ) -> TGResponse: + logger.debug("Sendind media group: %r (text=%r)", media, text) media_list: List[dict] = [] allowed_medias = {"image", "gifv", "video", "audio", "unknown"} for attachment in media: @@ -149,6 +158,7 @@ class TelegramIntegration(BaseIntegration): async def _post_poll( self, poll: Poll, reply_to: Optional[str] = None ) -> TGResponse: + logger.debug("Sending poll: %r", poll) return await self._tg_request( "sendPoll", disable_notification=self.silent, @@ -198,9 +208,17 @@ class TelegramIntegration(BaseIntegration): return str.join(",", map(str, ids)) def __repr__(self) -> str: + bot_uid, key = self.token.split(":") return ( "" - ).format(chat=self.chat_id, silent=self.silent, template=self.template) + ).format( + chat=self.chat_id, + silent=self.silent, + template=self.template, + bot_uid=bot_uid, + key=str.join("", ("X" for _ in key)), + ) From fd05f39f4aa750bae09b707dec5754190fc486f3 Mon Sep 17 00:00:00 2001 From: hkc Date: Tue, 1 Nov 2022 13:52:40 +0300 Subject: [PATCH 4/7] Changed default level to ERROR, added TG API logs --- mastoposter/__init__.py | 2 +- mastoposter/__main__.py | 12 +++++++++--- mastoposter/integrations/telegram.py | 6 +++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mastoposter/__init__.py b/mastoposter/__init__.py index 1c4c717..8217e28 100644 --- a/mastoposter/__init__.py +++ b/mastoposter/__init__.py @@ -65,7 +65,7 @@ def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]: async def execute_integrations( status: Status, sinks: List[FilteredIntegration] ) -> List[Optional[str]]: - logger.debug("Executing integrations...") + logger.info("Executing integrations...") return await gather( *[sink[0](status) for sink in sinks if run_filters(sink[1], status)], return_exceptions=True, diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index 96ebfc8..a5c4ae7 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from asyncio import run from configparser import ConfigParser, ExtendedInterpolation -from logging import DEBUG, Formatter, StreamHandler, getLogger +from logging import INFO, Formatter, StreamHandler, getLogger from sys import stdout from mastoposter import execute_integrations, load_integrations_from from mastoposter.integrations import FilteredIntegration @@ -19,9 +19,9 @@ VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials" logger = getLogger() -def init_logger(loglevel: int = DEBUG): +def init_logger(loglevel: int = INFO): stdout_handler = StreamHandler(stdout) - stdout_handler.setLevel(DEBUG) + stdout_handler.setLevel(loglevel) formatter = Formatter("[%(asctime)s][%(levelname)5s:%(name)s] %(message)s") stdout_handler.setFormatter(formatter) logger.addHandler(stdout_handler) @@ -39,6 +39,12 @@ async def listen( async for status in source(**kwargs): logger.debug("Got status: %r", status) if status.account.id != user: + logger.info( + "Skipping status %s (account.id=%r != %r)", + status.uri, + status.account.id, + user, + ) continue # TODO: add option/filter to handle that diff --git a/mastoposter/integrations/telegram.py b/mastoposter/integrations/telegram.py index 4108dd3..3a3f185 100644 --- a/mastoposter/integrations/telegram.py +++ b/mastoposter/integrations/telegram.py @@ -88,9 +88,13 @@ class TelegramIntegration(BaseIntegration): async def _tg_request(self, method: str, **kwargs) -> TGResponse: url = API_URL.format(self.token, method) async with AsyncClient() as client: - return TGResponse.from_dict( + response = TGResponse.from_dict( (await client.post(url, json=kwargs)).json(), kwargs ) + if not response.ok: + logger.error("TG error: %r", response.error) + logger.info("parameters: %r", kwargs) + return response async def _post_plaintext(self, text: str) -> TGResponse: logger.debug("Sending HTML message: %r", text) From bfa0534f6dcdf2ab53bd9443dee5ddc269c7e768 Mon Sep 17 00:00:00 2001 From: hkc Date: Tue, 1 Nov 2022 14:33:47 +0300 Subject: [PATCH 5/7] uh, more logging? --- mastoposter/__init__.py | 2 +- mastoposter/__main__.py | 4 +++- mastoposter/filters/__init__.py | 2 +- mastoposter/filters/combined.py | 11 ++++++++--- mastoposter/integrations/discord/__init__.py | 2 +- mastoposter/integrations/telegram.py | 9 +++++++-- mastoposter/utils.py | 7 +++++++ 7 files changed, 28 insertions(+), 9 deletions(-) diff --git a/mastoposter/__init__.py b/mastoposter/__init__.py index 8217e28..a815d03 100644 --- a/mastoposter/__init__.py +++ b/mastoposter/__init__.py @@ -12,7 +12,7 @@ from mastoposter.integrations import ( ) from mastoposter.types import Status -logger = getLogger(__name__) +logger = getLogger() def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]: diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index a5c4ae7..ab38d11 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -37,6 +37,7 @@ async def listen( ): logger.info("Starting listening...") async for status in source(**kwargs): + logger.info("New status: %s", status.uri) logger.debug("Got status: %r", status) if status.account.id != user: logger.info( @@ -91,7 +92,8 @@ def main(config_path: str): ) account = Account.from_dict(rq.json()) user_id = account.id - logger.info("account.id=%s", user_id) + + logger.info("account.id=%s", user_id) url = "wss://{}/api/v1/streaming".format(conf["main"]["instance"]) diff --git a/mastoposter/filters/__init__.py b/mastoposter/filters/__init__.py index 015245c..79fbe23 100644 --- a/mastoposter/filters/__init__.py +++ b/mastoposter/filters/__init__.py @@ -11,7 +11,7 @@ from mastoposter.filters.text import TextFilter # NOQA from mastoposter.filters.spoiler import SpoilerFilter # NOQA from mastoposter.filters.visibility import VisibilityFilter # NOQA -logger = getLogger(__name__) +logger = getLogger("filters") def run_filters(filters: List[FilterInstance], status: Status) -> bool: diff --git a/mastoposter/filters/combined.py b/mastoposter/filters/combined.py index 0700cd4..49f1174 100644 --- a/mastoposter/filters/combined.py +++ b/mastoposter/filters/combined.py @@ -12,7 +12,7 @@ class CombinedFilter(BaseFilter, filter_name="combined"): } def __init__(self, filter_names: List[str], operator: str): - self.filter_names = filter_names + self._filter_names = filter_names self._operator_name = operator self.operator = self.OPERATORS[self._operator_name] self.filters: List[FilterInstance] = [] @@ -30,7 +30,7 @@ class CombinedFilter(BaseFilter, filter_name="combined"): super().post_init(filters, config) self.filters = [ self.new_instance(name, config["filter/" + name.lstrip("~!")]) - for name in self.filter_names + for name in self._filter_names ] def __call__(self, post: Status) -> bool: @@ -39,7 +39,12 @@ class CombinedFilter(BaseFilter, filter_name="combined"): return self.operator([f[1](post) ^ f[0] for f in self.filters]) def __repr__(self): + if self.filters: + return ( + f"Filter:combined(op={self._operator_name}, " + f"filters={self.filters!r})" + ) return ( f"Filter:combined(op={self._operator_name}, " - f"filters={self.filters!r})" + f"filters={self._filter_names!r}, loaded=False)" ) diff --git a/mastoposter/integrations/discord/__init__.py b/mastoposter/integrations/discord/__init__.py index bc674d4..04d4d00 100644 --- a/mastoposter/integrations/discord/__init__.py +++ b/mastoposter/integrations/discord/__init__.py @@ -11,7 +11,7 @@ from mastoposter.integrations.discord.types import ( ) from mastoposter.types import Status -logger = getLogger(__name__) +logger = getLogger("integrations.discord") class DiscordIntegration(BaseIntegration): diff --git a/mastoposter/integrations/telegram.py b/mastoposter/integrations/telegram.py index 3a3f185..9b9889c 100644 --- a/mastoposter/integrations/telegram.py +++ b/mastoposter/integrations/telegram.py @@ -9,7 +9,7 @@ from mastoposter.types import Attachment, Poll, Status from emoji import emojize -logger = getLogger(__name__) +logger = getLogger("integrations.telegram") @dataclass @@ -93,7 +93,7 @@ class TelegramIntegration(BaseIntegration): ) if not response.ok: logger.error("TG error: %r", response.error) - logger.info("parameters: %r", kwargs) + logger.error("parameters: %r", kwargs) return response async def _post_plaintext(self, text: str) -> TGResponse: @@ -135,6 +135,11 @@ class TelegramIntegration(BaseIntegration): if attachment.type not in allowed_medias: continue if attachment.type not in MEDIA_COMPATIBILITY: + logger.warning( + "attachment %r is not in %r", + attachment.type, + MEDIA_COMPATIBILITY, + ) continue allowed_medias &= MEDIA_COMPATIBILITY[attachment.type] media_list.append( diff --git a/mastoposter/utils.py b/mastoposter/utils.py index 14e6bcc..3474908 100644 --- a/mastoposter/utils.py +++ b/mastoposter/utils.py @@ -1,8 +1,11 @@ from configparser import ConfigParser from html import escape +from logging import getLogger from typing import Callable, Dict from bs4.element import Tag, PageElement +logger = getLogger("utils") + def md_escape(text: str) -> str: return ( @@ -24,9 +27,13 @@ def normalize_config(conf: ConfigParser): normalized_key = k.replace(" ", "_").replace("-", "_") if k == normalized_key: continue + logger.info( + "moving %r.%r -> %r.%r", section, k, section, normalized_key + ) conf[section][normalized_key] = v _remove.add(k) for k in _remove: + logger.info("removing key %r.%r", section, k) del conf[section][k] From 94f477bc45e053a40b682aab923ec6ae417851ec Mon Sep 17 00:00:00 2001 From: hkc Date: Wed, 2 Nov 2022 20:11:38 +0300 Subject: [PATCH 6/7] Added configurable logging level --- README.md | 4 ++++ mastoposter/__main__.py | 4 ++-- mastoposter/sources.py | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 130ff1e..733104d 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ connection error!!!) #### modules More about them later +#### loglevel +Self-explanatory, logging level. Can be either `DEBUG`, `INFO`, `WARNING` or +`ERROR`. Defaults to `INFO` + ### Modules There's two types of modules supported at this point: `telegram` and `discord`. Both of them are self-explanatory, but we'll go over them real quick. diff --git a/mastoposter/__main__.py b/mastoposter/__main__.py index ab38d11..371d4ac 100644 --- a/mastoposter/__main__.py +++ b/mastoposter/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from asyncio import run from configparser import ConfigParser, ExtendedInterpolation -from logging import INFO, Formatter, StreamHandler, getLogger +from logging import INFO, Formatter, StreamHandler, getLevelName, getLogger from sys import stdout from mastoposter import execute_integrations, load_integrations_from from mastoposter.integrations import FilteredIntegration @@ -72,10 +72,10 @@ async def listen( def main(config_path: str): - init_logger() conf = ConfigParser(interpolation=ExtendedInterpolation()) conf.read(config_path) + init_logger(getLevelName(conf["main"].get("loglevel", "INFO"))) normalize_config(conf) modules: List[FilteredIntegration] = load_integrations_from(conf) diff --git a/mastoposter/sources.py b/mastoposter/sources.py index e2bdbb9..5865d46 100644 --- a/mastoposter/sources.py +++ b/mastoposter/sources.py @@ -1,10 +1,13 @@ from asyncio import exceptions from json import loads +from logging import getLogger from typing import AsyncGenerator from urllib.parse import urlencode from mastoposter.types import Status +logger = getLogger("sources") + async def websocket_source( url: str, reconnect: bool = False, **params @@ -22,6 +25,20 @@ async def websocket_source( raise Exception(event["error"]) if event["event"] == "update": yield Status.from_dict(loads(event["payload"])) - except (WebSocketException, TimeoutError, exceptions.TimeoutError): + else: + logger.warn("unknown event type %r", event["event"]) + logger.debug("data: %r", event) + except ( + WebSocketException, + TimeoutError, + exceptions.TimeoutError, + ) as e: if not reconnect: raise + else: + logger.warn("%r caught, reconnecting", e) + else: + logger.info( + "WebSocket closed connection without any errors, " + "but we're not done yet" + ) From 3904e475a4c59f5884b29c0c21d9c777d2202d94 Mon Sep 17 00:00:00 2001 From: hkc Date: Wed, 2 Nov 2022 20:18:31 +0300 Subject: [PATCH 7/7] MORE. LOGS. --- mastoposter/integrations/discord/__init__.py | 9 ++++++++- mastoposter/integrations/telegram.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mastoposter/integrations/discord/__init__.py b/mastoposter/integrations/discord/__init__.py index 04d4d00..dd88b66 100644 --- a/mastoposter/integrations/discord/__init__.py +++ b/mastoposter/integrations/discord/__init__.py @@ -41,12 +41,14 @@ class DiscordIntegration(BaseIntegration): logger.debug("Executing webhook with %r", json) - return ( + result = ( await c.post( self.webhook, json=json, ) ).json() + logger.debug("Result: %r", result) + return result async def __call__(self, status: Status) -> Optional[str]: source = status.reblog or status @@ -88,6 +90,11 @@ class DiscordIntegration(BaseIntegration): ), ) ) + else: + logger.warn( + "Unsupported attachment %r for Discord Embed", + attachment.type, + ) await self.execute_webhook( username=status.account.acct, diff --git a/mastoposter/integrations/telegram.py b/mastoposter/integrations/telegram.py index 9b9889c..50ac846 100644 --- a/mastoposter/integrations/telegram.py +++ b/mastoposter/integrations/telegram.py @@ -88,12 +88,15 @@ class TelegramIntegration(BaseIntegration): async def _tg_request(self, method: str, **kwargs) -> TGResponse: url = API_URL.format(self.token, method) async with AsyncClient() as client: + logger.debug("TG request: %s(%r)", method, kwargs) response = TGResponse.from_dict( (await client.post(url, json=kwargs)).json(), kwargs ) if not response.ok: logger.error("TG error: %r", response.error) logger.error("parameters: %r", kwargs) + else: + logger.debug("Result: %r", response.result) return response async def _post_plaintext(self, text: str) -> TGResponse: