Merge pull request #19 from hatkidchan/logging

Added logging functionality
This commit is contained in:
Casey 2022-11-02 20:30:19 +03:00 committed by GitHub
commit 5b7a4dec1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 310 additions and 77 deletions

View File

@ -104,6 +104,10 @@ connection error!!!)
#### modules #### modules
More about them later More about them later
#### loglevel
Self-explanatory, logging level. Can be either `DEBUG`, `INFO`, `WARNING` or
`ERROR`. Defaults to `INFO`
### Modules ### Modules
There's two types of modules supported at this point: `telegram` and `discord`. There's two types of modules supported at this point: `telegram` and `discord`.
Both of them are self-explanatory, but we'll go over them real quick. Both of them are self-explanatory, but we'll go over them real quick.
@ -175,15 +179,13 @@ expression to match your spoiler text.
Filter to match post content against either a regular expression, or a list of Filter to match post content against either a regular expression, or a list of
tags. Matching is done on the plaintext version of the post. tags. Matching is done on the plaintext version of the post.
`mode` property determines the type of operation. Can be either `regexp`, You can have one of two properties (but not both because fuck you): `tags` or
`tag` or `hashtag` (two last ones are the same). `regexp`. It's obvious what does what: if you have `regexp` set, plaintext
version of status is checked against that regular expression, and if you have
`tags` set, then only statuses that have those tags will be allowed.
In `mode = regexp` you have to specify the `regexp` option with a regular Please note that in case of tags, you should NOT use `#` symbol in front of
expression to match against. them.
In `mode = tag` or `mode = hashtag`, `tags` option should contain a
space-separated list of tags. If any of the tags are present in that list,
filter will be triggered.
#### `type = visibility` #### `type = visibility`
Simple filter that just checks for post visibility. Simple filter that just checks for post visibility.

View File

@ -1,5 +1,6 @@
from asyncio import gather from asyncio import gather
from configparser import ConfigParser from configparser import ConfigParser
from logging import getLogger
from typing import Dict, List, Optional from typing import Dict, List, Optional
from mastoposter.filters import run_filters from mastoposter.filters import run_filters
from mastoposter.filters.base import BaseFilter, FilterInstance from mastoposter.filters.base import BaseFilter, FilterInstance
@ -11,33 +12,49 @@ from mastoposter.integrations import (
) )
from mastoposter.types import Status from mastoposter.types import Status
logger = getLogger()
def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]: def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]:
modules: List[FilteredIntegration] = [] modules: List[FilteredIntegration] = []
for module_name in config.get("main", "modules").split(): for module_name in config.get("main", "modules").split():
mod = config[f"module/{module_name}"] mod = config[f"module/{module_name}"]
logger.info(
"Configuring %s integration (type=%s)", module_name, mod["type"]
)
filters: Dict[str, FilterInstance] = {} filters: Dict[str, FilterInstance] = {}
for filter_name in mod.get("filters", "").split(): for filter_name in mod.get("filters", "").split():
filter_basename = filter_name.lstrip("~!") filter_basename = filter_name.lstrip("~!")
logger.info(
"Loading filter %s for integration %s",
filter_basename,
module_name,
)
filters[filter_basename] = BaseFilter.new_instance( filters[filter_basename] = BaseFilter.new_instance(
filter_name, config[f"filter/{filter_basename}"] filter_name, config[f"filter/{filter_basename}"]
) )
for finst in list(filters.values()): for finst in list(filters.values()):
logger.info("Running post-initialization hook for %r", finst)
finst.filter.post_init(filters, config) finst.filter.post_init(filters, config)
# TODO: make a registry of integrations
# INFO: mastoposter/integrations/base.py@__init__
if mod["type"] == "telegram": if mod["type"] == "telegram":
modules.append( modules.append(
FilteredIntegration( FilteredIntegration(
TelegramIntegration(mod), list(filters.values()) TelegramIntegration.from_section(mod),
list(filters.values()),
) )
) )
elif mod["type"] == "discord": elif mod["type"] == "discord":
modules.append( modules.append(
FilteredIntegration( FilteredIntegration(
DiscordIntegration(mod), list(filters.values()) DiscordIntegration.from_section(mod),
list(filters.values()),
) )
) )
else: else:
@ -48,6 +65,7 @@ def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]:
async def execute_integrations( async def execute_integrations(
status: Status, sinks: List[FilteredIntegration] status: Status, sinks: List[FilteredIntegration]
) -> List[Optional[str]]: ) -> List[Optional[str]]:
logger.info("Executing integrations...")
return await gather( return await gather(
*[sink[0](status) for sink in sinks if run_filters(sink[1], status)], *[sink[0](status) for sink in sinks if run_filters(sink[1], status)],
return_exceptions=True, return_exceptions=True,

View File

@ -1,6 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from asyncio import run from asyncio import run
from configparser import ConfigParser, ExtendedInterpolation from configparser import ConfigParser, ExtendedInterpolation
from logging import INFO, Formatter, StreamHandler, getLevelName, getLogger
from sys import stdout
from mastoposter import execute_integrations, load_integrations_from from mastoposter import execute_integrations, load_integrations_from
from mastoposter.integrations import FilteredIntegration from mastoposter.integrations import FilteredIntegration
from mastoposter.sources import websocket_source from mastoposter.sources import websocket_source
@ -8,10 +10,23 @@ from typing import AsyncGenerator, Callable, List
from mastoposter.types import Account, Status from mastoposter.types import Account, Status
from httpx import Client from httpx import Client
from mastoposter.utils import normalize_config
WSOCK_TEMPLATE = "wss://{instance}/api/v1/streaming" WSOCK_TEMPLATE = "wss://{instance}/api/v1/streaming"
VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials" VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials"
logger = getLogger()
def init_logger(loglevel: int = INFO):
stdout_handler = StreamHandler(stdout)
stdout_handler.setLevel(loglevel)
formatter = Formatter("[%(asctime)s][%(levelname)5s:%(name)s] %(message)s")
stdout_handler.setFormatter(formatter)
logger.addHandler(stdout_handler)
logger.setLevel(loglevel)
async def listen( async def listen(
source: Callable[..., AsyncGenerator[Status, None]], source: Callable[..., AsyncGenerator[Status, None]],
@ -20,12 +35,26 @@ async def listen(
/, /,
**kwargs, **kwargs,
): ):
logger.info("Starting listening...")
async for status in source(**kwargs): async for status in source(**kwargs):
logger.info("New status: %s", status.uri)
logger.debug("Got status: %r", status)
if status.account.id != user: if status.account.id != user:
logger.info(
"Skipping status %s (account.id=%r != %r)",
status.uri,
status.account.id,
user,
)
continue continue
# TODO: add option/filter to handle that # TODO: add option/filter to handle that
if status.visibility in ("direct",): if status.visibility in ("direct",):
logger.info(
"Skipping post %s (status.visibility=%r)",
status.uri,
status.visibility,
)
continue continue
# TODO: find a better way to handle threads # TODO: find a better way to handle threads
@ -33,6 +62,10 @@ async def listen(
status.in_reply_to_account_id is not None status.in_reply_to_account_id is not None
and status.in_reply_to_account_id != user and status.in_reply_to_account_id != user
): ):
logger.info(
"Skipping post %s because it's a reply to another person",
status.uri,
)
continue continue
await execute_integrations(status, drains) await execute_integrations(status, drains)
@ -42,21 +75,16 @@ def main(config_path: str):
conf = ConfigParser(interpolation=ExtendedInterpolation()) conf = ConfigParser(interpolation=ExtendedInterpolation())
conf.read(config_path) conf.read(config_path)
for section in conf.sections(): init_logger(getLevelName(conf["main"].get("loglevel", "INFO")))
_remove = set() normalize_config(conf)
for k, v in conf[section].items():
normalized_key = k.replace(" ", "_").replace("-", "_")
if k == normalized_key:
continue
conf[section][normalized_key] = v
_remove.add(k)
for k in _remove:
del conf[section][k]
modules: List[FilteredIntegration] = load_integrations_from(conf) modules: List[FilteredIntegration] = load_integrations_from(conf)
logger.info("Loaded %d integrations", len(modules))
user_id: str = conf["main"]["user"] user_id: str = conf["main"]["user"]
if user_id == "auto": if user_id == "auto":
logger.info("config.main.user is set to auto, getting user ID")
with Client() as c: with Client() as c:
rq = c.get( rq = c.get(
VERIFY_CREDS_TEMPLATE.format(**conf["main"]), VERIFY_CREDS_TEMPLATE.format(**conf["main"]),
@ -65,7 +93,10 @@ def main(config_path: str):
account = Account.from_dict(rq.json()) account = Account.from_dict(rq.json())
user_id = account.id user_id = account.id
logger.info("account.id=%s", user_id)
url = "wss://{}/api/v1/streaming".format(conf["main"]["instance"]) url = "wss://{}/api/v1/streaming".format(conf["main"]["instance"])
run( run(
listen( listen(
websocket_source, websocket_source,

View File

@ -1,3 +1,4 @@
from logging import getLogger
from typing import List from typing import List
from mastoposter.types import Status from mastoposter.types import Status
@ -10,8 +11,26 @@ from mastoposter.filters.text import TextFilter # NOQA
from mastoposter.filters.spoiler import SpoilerFilter # NOQA from mastoposter.filters.spoiler import SpoilerFilter # NOQA
from mastoposter.filters.visibility import VisibilityFilter # NOQA from mastoposter.filters.visibility import VisibilityFilter # NOQA
logger = getLogger("filters")
def run_filters(filters: List[FilterInstance], status: Status) -> bool: def run_filters(filters: List[FilterInstance], status: Status) -> bool:
logger.debug("Running filters on %r", status.id)
if not filters: if not filters:
logger.debug("No filters, returning True")
return True return True
return all((fil.filter(status) ^ fil.inverse for fil in filters))
results: List[bool] = []
for fil in filters:
result = fil.filter(status)
logger.debug(
"%r -> %r ^ %r -> %r",
fil.filter,
result,
fil.inverse,
result ^ fil.inverse,
)
results.append(result ^ fil.inverse)
logger.debug("Result: %r", all(results))
return all(results)

View File

@ -23,8 +23,8 @@ class BaseFilter(ABC):
filter_name: ClassVar[str] = "_base" filter_name: ClassVar[str] = "_base"
def __init__(self, section: SectionProxy): def __init__(self):
UNUSED(section) pass
def __init_subclass__(cls, filter_name: str, **kwargs): def __init_subclass__(cls, filter_name: str, **kwargs):
super().__init_subclass__(**kwargs) super().__init_subclass__(**kwargs)
@ -51,7 +51,11 @@ class BaseFilter(ABC):
def load_filter(cls, name: str, section: SectionProxy) -> "BaseFilter": def load_filter(cls, name: str, section: SectionProxy) -> "BaseFilter":
if name not in cls.FILTER_REGISTRY: if name not in cls.FILTER_REGISTRY:
raise KeyError(f"no filter with name {name!r} was found") raise KeyError(f"no filter with name {name!r} was found")
return cls.FILTER_REGISTRY[name](section) return cls.FILTER_REGISTRY[name].from_section(section)
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseFilter":
raise NotImplementedError
@classmethod @classmethod
def new_instance(cls, name: str, section: SectionProxy) -> FilterInstance: def new_instance(cls, name: str, section: SectionProxy) -> FilterInstance:

View File

@ -1,13 +1,18 @@
from configparser import SectionProxy from configparser import SectionProxy
from fnmatch import fnmatch from fnmatch import fnmatch
from typing import List
from mastoposter.filters.base import BaseFilter from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status from mastoposter.types import Status
class BoostFilter(BaseFilter, filter_name="boost"): class BoostFilter(BaseFilter, filter_name="boost"):
def __init__(self, section: SectionProxy): def __init__(self, accounts: List[str]):
super().__init__(section) super().__init__()
self.list = section.get("list", "").split() self.list = accounts
@classmethod
def from_section(cls, section: SectionProxy) -> "BoostFilter":
return cls(section["list"].split())
@classmethod @classmethod
def check_account(cls, acct: str, mask: str) -> bool: def check_account(cls, acct: str, mask: str) -> bool:

View File

@ -11,19 +11,26 @@ class CombinedFilter(BaseFilter, filter_name="combined"):
"single": lambda d: sum(d) == 1, "single": lambda d: sum(d) == 1,
} }
def __init__(self, section: SectionProxy): def __init__(self, filter_names: List[str], operator: str):
self.filter_names = section.get("filters", "").split() self._filter_names = filter_names
self.operator = self.OPERATORS[section.get("operator", "all")] self._operator_name = operator
self._operator_name = section.get("operator", "all") self.operator = self.OPERATORS[self._operator_name]
self.filters: List[FilterInstance] = [] self.filters: List[FilterInstance] = []
@classmethod
def from_section(cls, section: SectionProxy) -> "CombinedFilter":
return cls(
filter_names=section["filters"].split(),
operator=section.get("operator", "all"),
)
def post_init( def post_init(
self, filters: Dict[str, FilterInstance], config: ConfigParser self, filters: Dict[str, FilterInstance], config: ConfigParser
): ):
super().post_init(filters, config) super().post_init(filters, config)
self.filters = [ self.filters = [
self.new_instance(name, config["filter/" + name.lstrip("~!")]) self.new_instance(name, config["filter/" + name.lstrip("~!")])
for name in self.filter_names for name in self._filter_names
] ]
def __call__(self, post: Status) -> bool: def __call__(self, post: Status) -> bool:
@ -32,7 +39,12 @@ class CombinedFilter(BaseFilter, filter_name="combined"):
return self.operator([f[1](post) ^ f[0] for f in self.filters]) return self.operator([f[1](post) ^ f[0] for f in self.filters])
def __repr__(self): def __repr__(self):
if self.filters:
return ( return (
f"Filter:combined(op={self._operator_name}, " f"Filter:combined(op={self._operator_name}, "
f"filters={self.filters!r})" f"filters={self.filters!r})"
) )
return (
f"Filter:combined(op={self._operator_name}, "
f"filters={self._filter_names!r}, loaded=False)"
)

View File

@ -1,16 +1,26 @@
from configparser import SectionProxy from configparser import SectionProxy
from typing import Set from typing import Literal, Set
from mastoposter.filters.base import BaseFilter from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status from mastoposter.types import Status
class MediaFilter(BaseFilter, filter_name="media"): class MediaFilter(BaseFilter, filter_name="media"):
def __init__(self, section: SectionProxy): def __init__(
super().__init__(section) self,
self.valid_media: Set[str] = set(section.get("valid_media").split()) valid_media: Set[str],
self.mode = section.get("mode", "include") mode: Literal["include", "exclude", "only"],
if self.mode not in ("include", "exclude", "only"): ):
raise ValueError(f"{self.mode=} is not valid") super().__init__()
self.valid_media: Set[str] = valid_media
self.mode = mode
assert self.mode in ("include", "exclude", "only")
@classmethod
def from_section(cls, section: SectionProxy) -> "MediaFilter":
return cls(
valid_media=set(section["valid_media"].split()),
mode=section.get("mode", "include"), # type: ignore
)
def __call__(self, status: Status) -> bool: def __call__(self, status: Status) -> bool:
if not status.media_attachments: if not status.media_attachments:

View File

@ -1,6 +1,6 @@
from configparser import SectionProxy from configparser import SectionProxy
from re import Pattern, compile as regexp from re import Pattern, compile as regexp
from typing import ClassVar from typing import ClassVar, Set
from fnmatch import fnmatch from fnmatch import fnmatch
from mastoposter.filters.base import BaseFilter from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status from mastoposter.types import Status
@ -9,22 +9,27 @@ from mastoposter.types import Status
class MentionFilter(BaseFilter, filter_name="mention"): class MentionFilter(BaseFilter, filter_name="mention"):
MENTION_REGEX: ClassVar[Pattern] = regexp(r"@([^@]+)(@([^@]+))?") MENTION_REGEX: ClassVar[Pattern] = regexp(r"@([^@]+)(@([^@]+))?")
def __init__(self, section: SectionProxy): def __init__(self, accounts: Set[str]):
super().__init__(section) super().__init__()
self.list = section.get("list", "").split() self.accounts = accounts
@classmethod
def from_section(cls, section: SectionProxy) -> "MentionFilter":
return cls(set(section.get("list", "").split()))
@classmethod @classmethod
def check_account(cls, acct: str, mask: str) -> bool: def check_account(cls, acct: str, mask: str) -> bool:
return fnmatch("@" + acct, mask) return fnmatch("@" + acct, mask)
def __call__(self, status: Status) -> bool: def __call__(self, status: Status) -> bool:
if not self.list and status.mentions: if not self.accounts and status.mentions:
return True return True
# XXX: make it better somehow. and faster. and stronger.
return any( return any(
( (
any( any(
self.check_account(mention.acct, mask) self.check_account(mention.acct, mask)
for mask in self.list for mask in self.accounts
) )
for mention in status.mentions for mention in status.mentions
) )
@ -32,5 +37,7 @@ class MentionFilter(BaseFilter, filter_name="mention"):
def __repr__(self): def __repr__(self):
return str.format( return str.format(
"Filter:{name}({list!r})", name=self.filter_name, list=self.list "Filter:{name}({list!r})",
name=self.filter_name,
list=self.accounts,
) )

View File

@ -5,9 +5,13 @@ from mastoposter.types import Status
class SpoilerFilter(BaseFilter, filter_name="spoiler"): class SpoilerFilter(BaseFilter, filter_name="spoiler"):
def __init__(self, section: SectionProxy): def __init__(self, regex: str = "^.*$"):
super().__init__(section) super().__init__()
self.regexp: Pattern = regexp(section.get("regexp", "^.*$")) self.regexp: Pattern = regexp(regex)
@classmethod
def from_section(cls, section: SectionProxy) -> "SpoilerFilter":
return cls(section.get("regexp", section.get("regex", "^.*$")))
def __call__(self, status: Status) -> bool: def __call__(self, status: Status) -> bool:
return self.regexp.match(status.spoiler_text) is not None return self.regexp.match(status.spoiler_text) is not None

View File

@ -8,18 +8,24 @@ from mastoposter.types import Status
class TextFilter(BaseFilter, filter_name="content"): class TextFilter(BaseFilter, filter_name="content"):
def __init__(self, section: SectionProxy): def __init__(
super().__init__(section) self, regex: Optional[str] = None, tags: Optional[Set[str]] = None
self.mode = section["mode"] ):
self.tags: Set[str] = set() super().__init__()
self.regexp: Optional[Pattern] = None assert regex is not None or tags
if self.mode == "regexp": self.tags: Optional[Set[str]] = tags
self.regexp = regexp(section["regexp"]) self.regexp: Optional[Pattern] = regexp(regex) if regex else None
elif self.mode in ("hashtag", "tag"):
self.tags = set(map(str.lower, section["tags"].split())) @classmethod
else: def from_section(cls, section: SectionProxy) -> "TextFilter":
raise ValueError(f"Invalid filter mode {self.mode}") if "regexp" in section and "tags" in section:
raise AssertionError("you can't use both tags and regexp")
elif "regexp" in section:
return cls(regex=section["regexp"])
elif "tags" in section:
return cls(tags=set(section["tags"].split()))
raise AssertionError("neither regexp or tags were set")
@classmethod @classmethod
def node_to_text(cls, el: PageElement) -> str: def node_to_text(cls, el: PageElement) -> str:
@ -63,3 +69,4 @@ class TextFilter(BaseFilter, filter_name="content"):
name=self.filter_name, name=self.filter_name,
tags=self.tags, tags=self.tags,
) )
return "Filter:{name}(invalid)".format(name=self.filter_name)

View File

@ -1,12 +1,17 @@
from configparser import SectionProxy from configparser import SectionProxy
from typing import Set
from mastoposter.filters.base import BaseFilter from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status from mastoposter.types import Status
class VisibilityFilter(BaseFilter, filter_name="visibility"): class VisibilityFilter(BaseFilter, filter_name="visibility"):
def __init__(self, section: SectionProxy): def __init__(self, options: Set[str]):
super().__init__(section) super().__init__()
self.options = set(section["options"].split()) self.options = options
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseFilter":
return cls(set(section["options"].split()))
def __call__(self, status: Status) -> bool: def __call__(self, status: Status) -> bool:
return status.visibility in self.options return status.visibility in self.options

View File

@ -6,9 +6,14 @@ from mastoposter.types import Status
class BaseIntegration(ABC): class BaseIntegration(ABC):
def __init__(self, section: SectionProxy): # TODO: make a registry of integrations
def __init__(self):
pass pass
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseIntegration":
raise NotImplementedError
@abstractmethod @abstractmethod
async def __call__(self, status: Status) -> Optional[str]: async def __call__(self, status: Status) -> Optional[str]:
raise NotImplementedError raise NotImplementedError

View File

@ -1,4 +1,5 @@
from configparser import SectionProxy from configparser import SectionProxy
from logging import getLogger
from typing import List, Optional from typing import List, Optional
from httpx import AsyncClient from httpx import AsyncClient
from zlib import crc32 from zlib import crc32
@ -10,10 +11,16 @@ from mastoposter.integrations.discord.types import (
) )
from mastoposter.types import Status from mastoposter.types import Status
logger = getLogger("integrations.discord")
class DiscordIntegration(BaseIntegration): class DiscordIntegration(BaseIntegration):
def __init__(self, section: SectionProxy): def __init__(self, webhook: str):
self.webhook = section.get("webhook", "") self.webhook = webhook
@classmethod
def from_section(cls, section: SectionProxy) -> "DiscordIntegration":
return cls(section["webhook"])
async def execute_webhook( async def execute_webhook(
self, self,
@ -31,12 +38,17 @@ class DiscordIntegration(BaseIntegration):
if embeds is not None if embeds is not None
else [], else [],
} }
return (
logger.debug("Executing webhook with %r", json)
result = (
await c.post( await c.post(
self.webhook, self.webhook,
json=json, json=json,
) )
).json() ).json()
logger.debug("Result: %r", result)
return result
async def __call__(self, status: Status) -> Optional[str]: async def __call__(self, status: Status) -> Optional[str]:
source = status.reblog or status source = status.reblog or status
@ -78,6 +90,11 @@ class DiscordIntegration(BaseIntegration):
), ),
) )
) )
else:
logger.warn(
"Unsupported attachment %r for Discord Embed",
attachment.type,
)
await self.execute_webhook( await self.execute_webhook(
username=status.account.acct, username=status.account.acct,

View File

@ -1,5 +1,6 @@
from configparser import SectionProxy from configparser import SectionProxy
from dataclasses import dataclass from dataclasses import dataclass
from logging import getLogger
from typing import Any, List, Mapping, Optional from typing import Any, List, Mapping, Optional
from httpx import AsyncClient from httpx import AsyncClient
from jinja2 import Template from jinja2 import Template
@ -8,6 +9,9 @@ from mastoposter.types import Attachment, Poll, Status
from emoji import emojize from emoji import emojize
logger = getLogger("integrations.telegram")
@dataclass @dataclass
class TGResponse: class TGResponse:
ok: bool ok: bool
@ -54,22 +58,49 @@ Boost from <a href="{{status.reblog.account.url}}">\
class TelegramIntegration(BaseIntegration): class TelegramIntegration(BaseIntegration):
def __init__(self, sect: SectionProxy): def __init__(
self.token = sect.get("token", "") self,
self.chat_id = sect.get("chat", "") token: str,
self.silent = sect.getboolean("silent", True) chat_id: str,
self.template: Template = Template( template: Optional[Template] = None,
emojize(sect.get("template", DEFAULT_TEMPLATE)) silent: bool = True,
):
self.token = token
self.chat_id = chat_id
self.silent = silent
if template is None:
self.template = Template(emojize(DEFAULT_TEMPLATE))
else:
self.template = template
@classmethod
def from_section(cls, section: SectionProxy) -> "TelegramIntegration":
return cls(
token=section["token"],
chat_id=section["chat"],
silent=section.getboolean("silent", True),
template=Template(
emojize(section.get("template", DEFAULT_TEMPLATE))
),
) )
async def _tg_request(self, method: str, **kwargs) -> TGResponse: async def _tg_request(self, method: str, **kwargs) -> TGResponse:
url = API_URL.format(self.token, method) url = API_URL.format(self.token, method)
async with AsyncClient() as client: async with AsyncClient() as client:
return TGResponse.from_dict( logger.debug("TG request: %s(%r)", method, kwargs)
response = TGResponse.from_dict(
(await client.post(url, json=kwargs)).json(), kwargs (await client.post(url, json=kwargs)).json(), kwargs
) )
if not response.ok:
logger.error("TG error: %r", response.error)
logger.error("parameters: %r", kwargs)
else:
logger.debug("Result: %r", response.result)
return response
async def _post_plaintext(self, text: str) -> TGResponse: async def _post_plaintext(self, text: str) -> TGResponse:
logger.debug("Sending HTML message: %r", text)
return await self._tg_request( return await self._tg_request(
"sendMessage", "sendMessage",
parse_mode="HTML", parse_mode="HTML",
@ -82,6 +113,9 @@ class TelegramIntegration(BaseIntegration):
async def _post_media(self, text: str, media: Attachment) -> TGResponse: async def _post_media(self, text: str, media: Attachment) -> TGResponse:
# Just to be safe # Just to be safe
if media.type not in MEDIA_MAPPING: if media.type not in MEDIA_MAPPING:
logger.warning(
"Media %r has unknown type, falling back to plaintext", media
)
return await self._post_plaintext(text) return await self._post_plaintext(text)
return await self._tg_request( return await self._tg_request(
@ -97,12 +131,18 @@ class TelegramIntegration(BaseIntegration):
async def _post_mediagroup( async def _post_mediagroup(
self, text: str, media: List[Attachment] self, text: str, media: List[Attachment]
) -> TGResponse: ) -> TGResponse:
logger.debug("Sendind media group: %r (text=%r)", media, text)
media_list: List[dict] = [] media_list: List[dict] = []
allowed_medias = {"image", "gifv", "video", "audio", "unknown"} allowed_medias = {"image", "gifv", "video", "audio", "unknown"}
for attachment in media: for attachment in media:
if attachment.type not in allowed_medias: if attachment.type not in allowed_medias:
continue continue
if attachment.type not in MEDIA_COMPATIBILITY: if attachment.type not in MEDIA_COMPATIBILITY:
logger.warning(
"attachment %r is not in %r",
attachment.type,
MEDIA_COMPATIBILITY,
)
continue continue
allowed_medias &= MEDIA_COMPATIBILITY[attachment.type] allowed_medias &= MEDIA_COMPATIBILITY[attachment.type]
media_list.append( media_list.append(
@ -130,6 +170,7 @@ class TelegramIntegration(BaseIntegration):
async def _post_poll( async def _post_poll(
self, poll: Poll, reply_to: Optional[str] = None self, poll: Poll, reply_to: Optional[str] = None
) -> TGResponse: ) -> TGResponse:
logger.debug("Sending poll: %r", poll)
return await self._tg_request( return await self._tg_request(
"sendPoll", "sendPoll",
disable_notification=self.silent, disable_notification=self.silent,
@ -179,9 +220,17 @@ class TelegramIntegration(BaseIntegration):
return str.join(",", map(str, ids)) return str.join(",", map(str, ids))
def __repr__(self) -> str: def __repr__(self) -> str:
bot_uid, key = self.token.split(":")
return ( return (
"<TelegramIntegration " "<TelegramIntegration "
"chat_id={chat!r} " "chat_id={chat!r} "
"template={template!r} " "template={template!r} "
"token={bot_uid}:{key} "
"silent={silent!r}>" "silent={silent!r}>"
).format(chat=self.chat_id, silent=self.silent, template=self.template) ).format(
chat=self.chat_id,
silent=self.silent,
template=self.template,
bot_uid=bot_uid,
key=str.join("", ("X" for _ in key)),
)

View File

@ -1,10 +1,13 @@
from asyncio import exceptions from asyncio import exceptions
from json import loads from json import loads
from logging import getLogger
from typing import AsyncGenerator from typing import AsyncGenerator
from urllib.parse import urlencode from urllib.parse import urlencode
from mastoposter.types import Status from mastoposter.types import Status
logger = getLogger("sources")
async def websocket_source( async def websocket_source(
url: str, reconnect: bool = False, **params url: str, reconnect: bool = False, **params
@ -22,6 +25,20 @@ async def websocket_source(
raise Exception(event["error"]) raise Exception(event["error"])
if event["event"] == "update": if event["event"] == "update":
yield Status.from_dict(loads(event["payload"])) yield Status.from_dict(loads(event["payload"]))
except (WebSocketException, TimeoutError, exceptions.TimeoutError): else:
logger.warn("unknown event type %r", event["event"])
logger.debug("data: %r", event)
except (
WebSocketException,
TimeoutError,
exceptions.TimeoutError,
) as e:
if not reconnect: if not reconnect:
raise raise
else:
logger.warn("%r caught, reconnecting", e)
else:
logger.info(
"WebSocket closed connection without any errors, "
"but we're not done yet"
)

View File

@ -1,7 +1,11 @@
from configparser import ConfigParser
from html import escape from html import escape
from logging import getLogger
from typing import Callable, Dict from typing import Callable, Dict
from bs4.element import Tag, PageElement from bs4.element import Tag, PageElement
logger = getLogger("utils")
def md_escape(text: str) -> str: def md_escape(text: str) -> str:
return ( return (
@ -16,6 +20,23 @@ def md_escape(text: str) -> str:
) )
def normalize_config(conf: ConfigParser):
for section in conf.sections():
_remove = set()
for k, v in conf[section].items():
normalized_key = k.replace(" ", "_").replace("-", "_")
if k == normalized_key:
continue
logger.info(
"moving %r.%r -> %r.%r", section, k, section, normalized_key
)
conf[section][normalized_key] = v
_remove.add(k)
for k in _remove:
logger.info("removing key %r.%r", section, k)
del conf[section][k]
def node_to_html(el: PageElement) -> str: def node_to_html(el: PageElement) -> str:
TAG_TRANSFORMS: Dict[str, Callable[[Tag,], str]] = { TAG_TRANSFORMS: Dict[str, Callable[[Tag,], str]] = {
"a": lambda tag: '<a href="{}">{}</a>'.format( "a": lambda tag: '<a href="{}">{}</a>'.format(