import json import os import pathlib import jinja2 from flask import Flask, g, make_response, render_template, request, send_from_directory, session, jsonify from flask.json.provider import JSONProvider from src.config import Configuration from src.lib.api import create_api_v2_client_error_response, TDAPIError app = Flask( __name__, static_folder=Configuration().webserver["static_folder"], template_folder=Configuration().webserver["template_folder"], ) if jinja_bytecode_cache_path := Configuration().webserver["jinja_bytecode_cache_path"]: pathlib.Path(jinja_bytecode_cache_path).mkdir(parents=True, exist_ok=True) app.jinja_env.bytecode_cache = jinja2.FileSystemBytecodeCache(jinja_bytecode_cache_path) if Configuration().open_telemetry_endpoint: from src.internals.tracing.tracing import open_telemetry_init open_telemetry_init(app, Configuration().open_telemetry_endpoint) import datetime import logging import re from datetime import timedelta from os import getenv, listdir from os.path import exists, join, splitext from typing import TypedDict from urllib.parse import urljoin, quote_plus import dateutil.parser import humanize import orjson from src.lib.account import is_logged_in, load_account from src.lib.notification import count_new_notifications from src.lib.post import get_fileserver_for_value from src.pages.api import api_bp from src.types.account import Account from src.utils.utils import ( freesites, paysites, render_page_data, sanitize_html, url_is_for_non_logged_file_extension, parse_int, ) app.url_map.strict_slashes = False app.register_blueprint(api_bp) app.config.update( dict( ENABLE_PASSWORD_VALIDATOR=True, ENABLE_LOGIN_RATE_LIMITING=True, SESSION_REFRESH_EACH_REQUEST=False, SESSION_COOKIE_SAMESITE="Lax", SECRET_KEY=Configuration().webserver["secret"], cache_TYPE="null" if Configuration().development_mode else "simple", CACHE_DEFAULT_TIMEOUT=None if Configuration().development_mode else 60, SEND_FILE_MAX_AGE_DEFAULT=0, TEMPLATES_AUTO_RELOAD=True if Configuration().development_mode else False, ) ) def file_url(file: TypedDict("FILE", {"hash": str, "ext": str})) -> str: file_hash = file["hash"] ext = file["ext"] fn = f"/data/{file_hash[0:2]}/{file_hash[2:4]}/{file_hash}.{ext}" fs = get_fileserver_for_value(fn) return fs + fn class ORJSONProvider(JSONProvider): def __init__(self, *args, **kwargs): self.options = kwargs super().__init__(*args, **kwargs) def loads(self, s, **kwargs): if "object_hook" in kwargs: return json.loads(s, **kwargs) return orjson.loads(s) def dumps(self, obj, **kwargs): return orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS).decode("utf-8") app.json = ORJSONProvider(app) def simple_datetime(obj: str | datetime.datetime) -> str: if isinstance(obj, str): obj = dateutil.parser.parse(obj) return obj.strftime("%Y-%m-%d %H:%M:%S") app.jinja_options = dict(trim_blocks=True, lstrip_blocks=True) app.jinja_env.globals.update(is_logged_in=is_logged_in) app.jinja_env.globals.update(render_page_data=render_page_data) app.jinja_env.filters["relative_date"] = lambda val: humanize.naturaltime(val) app.jinja_env.filters["simple_date"] = lambda dt: dt.strftime("%Y-%m-%d") # maybe change app.jinja_env.filters["simple_datetime"] = simple_datetime app.jinja_env.filters["regex_match"] = lambda val, rgx: re.search(rgx, val) app.jinja_env.filters["regex_find"] = lambda val, rgx: re.findall(rgx, val) app.jinja_env.filters["sanitize_html"] = sanitize_html app.jinja_env.filters["file_url"] = file_url # todo use this instead of "{{ fancard.server or '' }}{{ fancard.path }}" app.jinja_env.filters["quote_plus"] = lambda u: quote_plus(u or "") app.jinja_env.filters["debug"] = lambda u: print(u) or u app.jinja_env.filters["parse_int"] = lambda val: parse_int(val) if Configuration().webserver["logging"]: logging.basicConfig( filename="../kemono.log", level=logging.getLevelName(Configuration().webserver["logging"]), ) logging.getLogger("PIL").setLevel(logging.INFO) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR) if Configuration().sentry_dsn: import sentry_sdk from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.redis import RedisIntegration sentry_sdk.utils.MAX_STRING_LENGTH = 2048 sentry_sdk.serializer.MAX_EVENT_BYTES = 10**7 sentry_sdk.serializer.MAX_DATABAG_DEPTH = 8 sentry_sdk.serializer.MAX_DATABAG_BREADTH = 20 sentry_sdk.init( integrations=[FlaskIntegration(), RedisIntegration()], dsn=Configuration().sentry_dsn, release=os.getenv("GIT_COMMIT_HASH") or "NOT_FOUND", max_request_body_size="always", max_value_length=1024 * 4, attach_stacktrace=True, max_breadcrumbs=1000, send_default_pii=True, ) import src.internals.cache.redis as redis import src.internals.database.database as database database.init() redis.init() if exists(nondynamic_folder := join(Configuration().webserver["template_folder"].replace("..", "."), "nondynamic")): def nondynamic_view(page_file): def nondynamic_view_func(): if ".html" in page_file: return make_response( render_template( f"nondynamic/{page_file}", props=dict(), base=request.args.to_dict(), ), 200, ) else: return send_from_directory(nondynamic_folder, page_file) return nondynamic_view_func for page in listdir(nondynamic_folder): app.get(f"/{splitext(page)[0]}", endpoint=f"get_{splitext(page)[0]}")(nondynamic_view(page)) @app.before_request def do_init_stuff(): app.permanent_session_lifetime = timedelta(days=3650) g.page_data = {} g.request_start_time = datetime.datetime.now() g.freesites = freesites g.artists_or_creators = Configuration().webserver["ui"]["config"]["artists_or_creators"] g.paysite_list = Configuration().webserver["ui"]["config"]["paysite_list"] g.paysites = paysites g.origin = Configuration().webserver["site"] g.custom_links = Configuration().webserver["ui"]["sidebar_items"] g.custom_footer = Configuration().webserver["ui"]["footer_items"] # Matomo. g.matomo_enabled = Configuration().webserver["ui"]["matomo"]["enabled"] g.matomo_plain_code = Configuration().webserver["ui"]["matomo"]["plain_code"] g.matomo_domain = Configuration().webserver["ui"]["matomo"]["tracking_domain"] g.matomo_code = Configuration().webserver["ui"]["matomo"]["tracking_code"] g.matomo_site_id = Configuration().webserver["ui"]["matomo"]["site_id"] # navbar g.disable_filehaus = Configuration().webserver["ui"]["sidebar"]["disable_filehaus"] g.disable_faq = Configuration().webserver["ui"]["sidebar"]["disable_faq"] g.disable_dms = Configuration().webserver["ui"]["sidebar"]["disable_dms"] # Ads. g.header_ad = Configuration().webserver["ui"]["ads"]["header"] g.middle_ad = Configuration().webserver["ui"]["ads"]["middle"] g.footer_ad = Configuration().webserver["ui"]["ads"]["footer"] g.slider_ad = Configuration().webserver["ui"]["ads"]["slider"] g.video_ad = Configuration().webserver["ui"]["ads"]["video"] # Banners. g.banner_global = Configuration().webserver["ui"]["banner"]["global"] g.banner_welcome = Configuration().webserver["ui"]["banner"]["welcome"] # Branding path prepend g.icons_prepend = Configuration().webserver["ui"]["files_url_prepend"]["icons_base_url"] g.banners_prepend = Configuration().webserver["ui"]["files_url_prepend"]["banners_base_url"] g.thumbnails_prepend = Configuration().webserver["ui"]["files_url_prepend"]["thumbnails_base_url"] g.mascot_path = Configuration().webserver["ui"]["home"]["mascot_path"] g.logo_path = Configuration().webserver["ui"]["home"]["logo_path"] g.welcome_credits = Configuration().webserver["ui"]["home"]["welcome_credits"] g.home_background_image = Configuration().webserver["ui"]["home"]["home_background_image"] g.announcements = Configuration().webserver["ui"]["home"]["announcements"] g.site_name = Configuration().webserver["ui"]["home"]["site_name"] g.canonical_url = urljoin(Configuration().webserver["site"], request.path) session.permanent = True session.modified = False if account := load_account(): g.account = account if Configuration().enable_notifications: g.new_notifications_count = count_new_notifications(g.account.id) @app.after_request def do_finish_stuff(response): if not url_is_for_non_logged_file_extension(request.path): start_time = g.request_start_time end_time = datetime.datetime.now() elapsed = end_time - start_time app.logger.debug( f"[{end_time.strftime("%Y-%m-%d %X")}] " f"Completed {request.method} request to {request.url} " f"in {elapsed.microseconds / 1000}ms" ) response.autocorrect_location_header = False return response # adding the handlers there because # flask doesn't allow handling 404/405 in blueprints # https://flask.palletsprojects.com/en/2.3.x/errorhandling/#blueprint-error-handlers @app.errorhandler(404) def route_not_found(error): if request.path.startswith('/api/v1'): return jsonify(error="Not Found"), 404 if request.path.startswith('/api/v2'): error = TDAPIError(type="http_error", message="Not Found") return create_api_v2_client_error_response(error, 404) return error, 404 @app.errorhandler(405) def method_not_allowed(error): if request.path.startswith('/api/v1'): return jsonify(error="Method Not Allowed"), 405 if request.path.startswith('/api/v2'): error = TDAPIError(type="http_error", message="Method Not Allowed") return create_api_v2_client_error_response(error, 405) return error, 405 @app.errorhandler(413) def upload_exceeded(error): props = {"redirect": request.headers.get("Referer") if request.headers.get("Referer") else "/"} limit = int(getenv("REQUESTS_IMAGES")) if getenv("REQUESTS_IMAGES") else 1048576 props["message"] = f"Submitted file exceeds the upload limit. {limit / 1024 / 1024} MB for requests images." return render_template("error.html", props=props), 413 @app.teardown_appcontext def close(e): # removing account just in case g.pop("account", None) cursor = g.pop("cursor", None) if cursor is not None: cursor.close() connection = g.pop("connection", None) if connection is not None: try: pool = database.get_pool() if not connection.autocommit: connection.commit() pool.putconn(connection) except Exception: pass