Merge pull request #19 from hatkidchan/logging

Added logging functionality
This commit is contained in:
Casey 2022-11-02 20:30:19 +03:00 committed by GitHub
commit 5b7a4dec1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 310 additions and 77 deletions

View File

@ -104,6 +104,10 @@ connection error!!!)
#### modules
More about them later
#### loglevel
Self-explanatory, logging level. Can be either `DEBUG`, `INFO`, `WARNING` or
`ERROR`. Defaults to `INFO`
### Modules
There's two types of modules supported at this point: `telegram` and `discord`.
Both of them are self-explanatory, but we'll go over them real quick.
@ -175,15 +179,13 @@ expression to match your spoiler text.
Filter to match post content against either a regular expression, or a list of
tags. Matching is done on the plaintext version of the post.
`mode` property determines the type of operation. Can be either `regexp`,
`tag` or `hashtag` (two last ones are the same).
You can have one of two properties (but not both because fuck you): `tags` or
`regexp`. It's obvious what does what: if you have `regexp` set, plaintext
version of status is checked against that regular expression, and if you have
`tags` set, then only statuses that have those tags will be allowed.
In `mode = regexp` you have to specify the `regexp` option with a regular
expression to match against.
In `mode = tag` or `mode = hashtag`, `tags` option should contain a
space-separated list of tags. If any of the tags are present in that list,
filter will be triggered.
Please note that in case of tags, you should NOT use `#` symbol in front of
them.
#### `type = visibility`
Simple filter that just checks for post visibility.

View File

@ -1,5 +1,6 @@
from asyncio import gather
from configparser import ConfigParser
from logging import getLogger
from typing import Dict, List, Optional
from mastoposter.filters import run_filters
from mastoposter.filters.base import BaseFilter, FilterInstance
@ -11,33 +12,49 @@ from mastoposter.integrations import (
)
from mastoposter.types import Status
logger = getLogger()
def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]:
modules: List[FilteredIntegration] = []
for module_name in config.get("main", "modules").split():
mod = config[f"module/{module_name}"]
logger.info(
"Configuring %s integration (type=%s)", module_name, mod["type"]
)
filters: Dict[str, FilterInstance] = {}
for filter_name in mod.get("filters", "").split():
filter_basename = filter_name.lstrip("~!")
logger.info(
"Loading filter %s for integration %s",
filter_basename,
module_name,
)
filters[filter_basename] = BaseFilter.new_instance(
filter_name, config[f"filter/{filter_basename}"]
)
for finst in list(filters.values()):
logger.info("Running post-initialization hook for %r", finst)
finst.filter.post_init(filters, config)
# TODO: make a registry of integrations
# INFO: mastoposter/integrations/base.py@__init__
if mod["type"] == "telegram":
modules.append(
FilteredIntegration(
TelegramIntegration(mod), list(filters.values())
TelegramIntegration.from_section(mod),
list(filters.values()),
)
)
elif mod["type"] == "discord":
modules.append(
FilteredIntegration(
DiscordIntegration(mod), list(filters.values())
DiscordIntegration.from_section(mod),
list(filters.values()),
)
)
else:
@ -48,6 +65,7 @@ def load_integrations_from(config: ConfigParser) -> List[FilteredIntegration]:
async def execute_integrations(
status: Status, sinks: List[FilteredIntegration]
) -> List[Optional[str]]:
logger.info("Executing integrations...")
return await gather(
*[sink[0](status) for sink in sinks if run_filters(sink[1], status)],
return_exceptions=True,

View File

@ -1,6 +1,8 @@
#!/usr/bin/env python3
from asyncio import run
from configparser import ConfigParser, ExtendedInterpolation
from logging import INFO, Formatter, StreamHandler, getLevelName, getLogger
from sys import stdout
from mastoposter import execute_integrations, load_integrations_from
from mastoposter.integrations import FilteredIntegration
from mastoposter.sources import websocket_source
@ -8,10 +10,23 @@ from typing import AsyncGenerator, Callable, List
from mastoposter.types import Account, Status
from httpx import Client
from mastoposter.utils import normalize_config
WSOCK_TEMPLATE = "wss://{instance}/api/v1/streaming"
VERIFY_CREDS_TEMPLATE = "https://{instance}/api/v1/accounts/verify_credentials"
logger = getLogger()
def init_logger(loglevel: int = INFO):
stdout_handler = StreamHandler(stdout)
stdout_handler.setLevel(loglevel)
formatter = Formatter("[%(asctime)s][%(levelname)5s:%(name)s] %(message)s")
stdout_handler.setFormatter(formatter)
logger.addHandler(stdout_handler)
logger.setLevel(loglevel)
async def listen(
source: Callable[..., AsyncGenerator[Status, None]],
@ -20,12 +35,26 @@ async def listen(
/,
**kwargs,
):
logger.info("Starting listening...")
async for status in source(**kwargs):
logger.info("New status: %s", status.uri)
logger.debug("Got status: %r", status)
if status.account.id != user:
logger.info(
"Skipping status %s (account.id=%r != %r)",
status.uri,
status.account.id,
user,
)
continue
# TODO: add option/filter to handle that
if status.visibility in ("direct",):
logger.info(
"Skipping post %s (status.visibility=%r)",
status.uri,
status.visibility,
)
continue
# TODO: find a better way to handle threads
@ -33,6 +62,10 @@ async def listen(
status.in_reply_to_account_id is not None
and status.in_reply_to_account_id != user
):
logger.info(
"Skipping post %s because it's a reply to another person",
status.uri,
)
continue
await execute_integrations(status, drains)
@ -42,21 +75,16 @@ def main(config_path: str):
conf = ConfigParser(interpolation=ExtendedInterpolation())
conf.read(config_path)
for section in conf.sections():
_remove = set()
for k, v in conf[section].items():
normalized_key = k.replace(" ", "_").replace("-", "_")
if k == normalized_key:
continue
conf[section][normalized_key] = v
_remove.add(k)
for k in _remove:
del conf[section][k]
init_logger(getLevelName(conf["main"].get("loglevel", "INFO")))
normalize_config(conf)
modules: List[FilteredIntegration] = load_integrations_from(conf)
logger.info("Loaded %d integrations", len(modules))
user_id: str = conf["main"]["user"]
if user_id == "auto":
logger.info("config.main.user is set to auto, getting user ID")
with Client() as c:
rq = c.get(
VERIFY_CREDS_TEMPLATE.format(**conf["main"]),
@ -65,7 +93,10 @@ def main(config_path: str):
account = Account.from_dict(rq.json())
user_id = account.id
logger.info("account.id=%s", user_id)
url = "wss://{}/api/v1/streaming".format(conf["main"]["instance"])
run(
listen(
websocket_source,

View File

@ -1,3 +1,4 @@
from logging import getLogger
from typing import List
from mastoposter.types import Status
@ -10,8 +11,26 @@ from mastoposter.filters.text import TextFilter # NOQA
from mastoposter.filters.spoiler import SpoilerFilter # NOQA
from mastoposter.filters.visibility import VisibilityFilter # NOQA
logger = getLogger("filters")
def run_filters(filters: List[FilterInstance], status: Status) -> bool:
logger.debug("Running filters on %r", status.id)
if not filters:
logger.debug("No filters, returning True")
return True
return all((fil.filter(status) ^ fil.inverse for fil in filters))
results: List[bool] = []
for fil in filters:
result = fil.filter(status)
logger.debug(
"%r -> %r ^ %r -> %r",
fil.filter,
result,
fil.inverse,
result ^ fil.inverse,
)
results.append(result ^ fil.inverse)
logger.debug("Result: %r", all(results))
return all(results)

View File

@ -23,8 +23,8 @@ class BaseFilter(ABC):
filter_name: ClassVar[str] = "_base"
def __init__(self, section: SectionProxy):
UNUSED(section)
def __init__(self):
pass
def __init_subclass__(cls, filter_name: str, **kwargs):
super().__init_subclass__(**kwargs)
@ -51,7 +51,11 @@ class BaseFilter(ABC):
def load_filter(cls, name: str, section: SectionProxy) -> "BaseFilter":
if name not in cls.FILTER_REGISTRY:
raise KeyError(f"no filter with name {name!r} was found")
return cls.FILTER_REGISTRY[name](section)
return cls.FILTER_REGISTRY[name].from_section(section)
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseFilter":
raise NotImplementedError
@classmethod
def new_instance(cls, name: str, section: SectionProxy) -> FilterInstance:

View File

@ -1,13 +1,18 @@
from configparser import SectionProxy
from fnmatch import fnmatch
from typing import List
from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status
class BoostFilter(BaseFilter, filter_name="boost"):
def __init__(self, section: SectionProxy):
super().__init__(section)
self.list = section.get("list", "").split()
def __init__(self, accounts: List[str]):
super().__init__()
self.list = accounts
@classmethod
def from_section(cls, section: SectionProxy) -> "BoostFilter":
return cls(section["list"].split())
@classmethod
def check_account(cls, acct: str, mask: str) -> bool:

View File

@ -11,19 +11,26 @@ class CombinedFilter(BaseFilter, filter_name="combined"):
"single": lambda d: sum(d) == 1,
}
def __init__(self, section: SectionProxy):
self.filter_names = section.get("filters", "").split()
self.operator = self.OPERATORS[section.get("operator", "all")]
self._operator_name = section.get("operator", "all")
def __init__(self, filter_names: List[str], operator: str):
self._filter_names = filter_names
self._operator_name = operator
self.operator = self.OPERATORS[self._operator_name]
self.filters: List[FilterInstance] = []
@classmethod
def from_section(cls, section: SectionProxy) -> "CombinedFilter":
return cls(
filter_names=section["filters"].split(),
operator=section.get("operator", "all"),
)
def post_init(
self, filters: Dict[str, FilterInstance], config: ConfigParser
):
super().post_init(filters, config)
self.filters = [
self.new_instance(name, config["filter/" + name.lstrip("~!")])
for name in self.filter_names
for name in self._filter_names
]
def __call__(self, post: Status) -> bool:
@ -32,7 +39,12 @@ class CombinedFilter(BaseFilter, filter_name="combined"):
return self.operator([f[1](post) ^ f[0] for f in self.filters])
def __repr__(self):
if self.filters:
return (
f"Filter:combined(op={self._operator_name}, "
f"filters={self.filters!r})"
)
return (
f"Filter:combined(op={self._operator_name}, "
f"filters={self._filter_names!r}, loaded=False)"
)

View File

@ -1,16 +1,26 @@
from configparser import SectionProxy
from typing import Set
from typing import Literal, Set
from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status
class MediaFilter(BaseFilter, filter_name="media"):
def __init__(self, section: SectionProxy):
super().__init__(section)
self.valid_media: Set[str] = set(section.get("valid_media").split())
self.mode = section.get("mode", "include")
if self.mode not in ("include", "exclude", "only"):
raise ValueError(f"{self.mode=} is not valid")
def __init__(
self,
valid_media: Set[str],
mode: Literal["include", "exclude", "only"],
):
super().__init__()
self.valid_media: Set[str] = valid_media
self.mode = mode
assert self.mode in ("include", "exclude", "only")
@classmethod
def from_section(cls, section: SectionProxy) -> "MediaFilter":
return cls(
valid_media=set(section["valid_media"].split()),
mode=section.get("mode", "include"), # type: ignore
)
def __call__(self, status: Status) -> bool:
if not status.media_attachments:

View File

@ -1,6 +1,6 @@
from configparser import SectionProxy
from re import Pattern, compile as regexp
from typing import ClassVar
from typing import ClassVar, Set
from fnmatch import fnmatch
from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status
@ -9,22 +9,27 @@ from mastoposter.types import Status
class MentionFilter(BaseFilter, filter_name="mention"):
MENTION_REGEX: ClassVar[Pattern] = regexp(r"@([^@]+)(@([^@]+))?")
def __init__(self, section: SectionProxy):
super().__init__(section)
self.list = section.get("list", "").split()
def __init__(self, accounts: Set[str]):
super().__init__()
self.accounts = accounts
@classmethod
def from_section(cls, section: SectionProxy) -> "MentionFilter":
return cls(set(section.get("list", "").split()))
@classmethod
def check_account(cls, acct: str, mask: str) -> bool:
return fnmatch("@" + acct, mask)
def __call__(self, status: Status) -> bool:
if not self.list and status.mentions:
if not self.accounts and status.mentions:
return True
# XXX: make it better somehow. and faster. and stronger.
return any(
(
any(
self.check_account(mention.acct, mask)
for mask in self.list
for mask in self.accounts
)
for mention in status.mentions
)
@ -32,5 +37,7 @@ class MentionFilter(BaseFilter, filter_name="mention"):
def __repr__(self):
return str.format(
"Filter:{name}({list!r})", name=self.filter_name, list=self.list
"Filter:{name}({list!r})",
name=self.filter_name,
list=self.accounts,
)

View File

@ -5,9 +5,13 @@ from mastoposter.types import Status
class SpoilerFilter(BaseFilter, filter_name="spoiler"):
def __init__(self, section: SectionProxy):
super().__init__(section)
self.regexp: Pattern = regexp(section.get("regexp", "^.*$"))
def __init__(self, regex: str = "^.*$"):
super().__init__()
self.regexp: Pattern = regexp(regex)
@classmethod
def from_section(cls, section: SectionProxy) -> "SpoilerFilter":
return cls(section.get("regexp", section.get("regex", "^.*$")))
def __call__(self, status: Status) -> bool:
return self.regexp.match(status.spoiler_text) is not None

View File

@ -8,18 +8,24 @@ from mastoposter.types import Status
class TextFilter(BaseFilter, filter_name="content"):
def __init__(self, section: SectionProxy):
super().__init__(section)
self.mode = section["mode"]
self.tags: Set[str] = set()
self.regexp: Optional[Pattern] = None
def __init__(
self, regex: Optional[str] = None, tags: Optional[Set[str]] = None
):
super().__init__()
assert regex is not None or tags
if self.mode == "regexp":
self.regexp = regexp(section["regexp"])
elif self.mode in ("hashtag", "tag"):
self.tags = set(map(str.lower, section["tags"].split()))
else:
raise ValueError(f"Invalid filter mode {self.mode}")
self.tags: Optional[Set[str]] = tags
self.regexp: Optional[Pattern] = regexp(regex) if regex else None
@classmethod
def from_section(cls, section: SectionProxy) -> "TextFilter":
if "regexp" in section and "tags" in section:
raise AssertionError("you can't use both tags and regexp")
elif "regexp" in section:
return cls(regex=section["regexp"])
elif "tags" in section:
return cls(tags=set(section["tags"].split()))
raise AssertionError("neither regexp or tags were set")
@classmethod
def node_to_text(cls, el: PageElement) -> str:
@ -63,3 +69,4 @@ class TextFilter(BaseFilter, filter_name="content"):
name=self.filter_name,
tags=self.tags,
)
return "Filter:{name}(invalid)".format(name=self.filter_name)

View File

@ -1,12 +1,17 @@
from configparser import SectionProxy
from typing import Set
from mastoposter.filters.base import BaseFilter
from mastoposter.types import Status
class VisibilityFilter(BaseFilter, filter_name="visibility"):
def __init__(self, section: SectionProxy):
super().__init__(section)
self.options = set(section["options"].split())
def __init__(self, options: Set[str]):
super().__init__()
self.options = options
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseFilter":
return cls(set(section["options"].split()))
def __call__(self, status: Status) -> bool:
return status.visibility in self.options

View File

@ -6,9 +6,14 @@ from mastoposter.types import Status
class BaseIntegration(ABC):
def __init__(self, section: SectionProxy):
# TODO: make a registry of integrations
def __init__(self):
pass
@classmethod
def from_section(cls, section: SectionProxy) -> "BaseIntegration":
raise NotImplementedError
@abstractmethod
async def __call__(self, status: Status) -> Optional[str]:
raise NotImplementedError

View File

@ -1,4 +1,5 @@
from configparser import SectionProxy
from logging import getLogger
from typing import List, Optional
from httpx import AsyncClient
from zlib import crc32
@ -10,10 +11,16 @@ from mastoposter.integrations.discord.types import (
)
from mastoposter.types import Status
logger = getLogger("integrations.discord")
class DiscordIntegration(BaseIntegration):
def __init__(self, section: SectionProxy):
self.webhook = section.get("webhook", "")
def __init__(self, webhook: str):
self.webhook = webhook
@classmethod
def from_section(cls, section: SectionProxy) -> "DiscordIntegration":
return cls(section["webhook"])
async def execute_webhook(
self,
@ -31,12 +38,17 @@ class DiscordIntegration(BaseIntegration):
if embeds is not None
else [],
}
return (
logger.debug("Executing webhook with %r", json)
result = (
await c.post(
self.webhook,
json=json,
)
).json()
logger.debug("Result: %r", result)
return result
async def __call__(self, status: Status) -> Optional[str]:
source = status.reblog or status
@ -78,6 +90,11 @@ class DiscordIntegration(BaseIntegration):
),
)
)
else:
logger.warn(
"Unsupported attachment %r for Discord Embed",
attachment.type,
)
await self.execute_webhook(
username=status.account.acct,

View File

@ -1,5 +1,6 @@
from configparser import SectionProxy
from dataclasses import dataclass
from logging import getLogger
from typing import Any, List, Mapping, Optional
from httpx import AsyncClient
from jinja2 import Template
@ -8,6 +9,9 @@ from mastoposter.types import Attachment, Poll, Status
from emoji import emojize
logger = getLogger("integrations.telegram")
@dataclass
class TGResponse:
ok: bool
@ -54,22 +58,49 @@ Boost from <a href="{{status.reblog.account.url}}">\
class TelegramIntegration(BaseIntegration):
def __init__(self, sect: SectionProxy):
self.token = sect.get("token", "")
self.chat_id = sect.get("chat", "")
self.silent = sect.getboolean("silent", True)
self.template: Template = Template(
emojize(sect.get("template", DEFAULT_TEMPLATE))
def __init__(
self,
token: str,
chat_id: str,
template: Optional[Template] = None,
silent: bool = True,
):
self.token = token
self.chat_id = chat_id
self.silent = silent
if template is None:
self.template = Template(emojize(DEFAULT_TEMPLATE))
else:
self.template = template
@classmethod
def from_section(cls, section: SectionProxy) -> "TelegramIntegration":
return cls(
token=section["token"],
chat_id=section["chat"],
silent=section.getboolean("silent", True),
template=Template(
emojize(section.get("template", DEFAULT_TEMPLATE))
),
)
async def _tg_request(self, method: str, **kwargs) -> TGResponse:
url = API_URL.format(self.token, method)
async with AsyncClient() as client:
return TGResponse.from_dict(
logger.debug("TG request: %s(%r)", method, kwargs)
response = TGResponse.from_dict(
(await client.post(url, json=kwargs)).json(), kwargs
)
if not response.ok:
logger.error("TG error: %r", response.error)
logger.error("parameters: %r", kwargs)
else:
logger.debug("Result: %r", response.result)
return response
async def _post_plaintext(self, text: str) -> TGResponse:
logger.debug("Sending HTML message: %r", text)
return await self._tg_request(
"sendMessage",
parse_mode="HTML",
@ -82,6 +113,9 @@ class TelegramIntegration(BaseIntegration):
async def _post_media(self, text: str, media: Attachment) -> TGResponse:
# Just to be safe
if media.type not in MEDIA_MAPPING:
logger.warning(
"Media %r has unknown type, falling back to plaintext", media
)
return await self._post_plaintext(text)
return await self._tg_request(
@ -97,12 +131,18 @@ class TelegramIntegration(BaseIntegration):
async def _post_mediagroup(
self, text: str, media: List[Attachment]
) -> TGResponse:
logger.debug("Sendind media group: %r (text=%r)", media, text)
media_list: List[dict] = []
allowed_medias = {"image", "gifv", "video", "audio", "unknown"}
for attachment in media:
if attachment.type not in allowed_medias:
continue
if attachment.type not in MEDIA_COMPATIBILITY:
logger.warning(
"attachment %r is not in %r",
attachment.type,
MEDIA_COMPATIBILITY,
)
continue
allowed_medias &= MEDIA_COMPATIBILITY[attachment.type]
media_list.append(
@ -130,6 +170,7 @@ class TelegramIntegration(BaseIntegration):
async def _post_poll(
self, poll: Poll, reply_to: Optional[str] = None
) -> TGResponse:
logger.debug("Sending poll: %r", poll)
return await self._tg_request(
"sendPoll",
disable_notification=self.silent,
@ -179,9 +220,17 @@ class TelegramIntegration(BaseIntegration):
return str.join(",", map(str, ids))
def __repr__(self) -> str:
bot_uid, key = self.token.split(":")
return (
"<TelegramIntegration "
"chat_id={chat!r} "
"template={template!r} "
"token={bot_uid}:{key} "
"silent={silent!r}>"
).format(chat=self.chat_id, silent=self.silent, template=self.template)
).format(
chat=self.chat_id,
silent=self.silent,
template=self.template,
bot_uid=bot_uid,
key=str.join("", ("X" for _ in key)),
)

View File

@ -1,10 +1,13 @@
from asyncio import exceptions
from json import loads
from logging import getLogger
from typing import AsyncGenerator
from urllib.parse import urlencode
from mastoposter.types import Status
logger = getLogger("sources")
async def websocket_source(
url: str, reconnect: bool = False, **params
@ -22,6 +25,20 @@ async def websocket_source(
raise Exception(event["error"])
if event["event"] == "update":
yield Status.from_dict(loads(event["payload"]))
except (WebSocketException, TimeoutError, exceptions.TimeoutError):
else:
logger.warn("unknown event type %r", event["event"])
logger.debug("data: %r", event)
except (
WebSocketException,
TimeoutError,
exceptions.TimeoutError,
) as e:
if not reconnect:
raise
else:
logger.warn("%r caught, reconnecting", e)
else:
logger.info(
"WebSocket closed connection without any errors, "
"but we're not done yet"
)

View File

@ -1,7 +1,11 @@
from configparser import ConfigParser
from html import escape
from logging import getLogger
from typing import Callable, Dict
from bs4.element import Tag, PageElement
logger = getLogger("utils")
def md_escape(text: str) -> str:
return (
@ -16,6 +20,23 @@ def md_escape(text: str) -> str:
)
def normalize_config(conf: ConfigParser):
for section in conf.sections():
_remove = set()
for k, v in conf[section].items():
normalized_key = k.replace(" ", "_").replace("-", "_")
if k == normalized_key:
continue
logger.info(
"moving %r.%r -> %r.%r", section, k, section, normalized_key
)
conf[section][normalized_key] = v
_remove.add(k)
for k in _remove:
logger.info("removing key %r.%r", section, k)
del conf[section][k]
def node_to_html(el: PageElement) -> str:
TAG_TRANSFORMS: Dict[str, Callable[[Tag,], str]] = {
"a": lambda tag: '<a href="{}">{}</a>'.format(