Commit de576ae3 authored by Lukáš Lalinský's avatar Lukáš Lalinský

Add gunicorn support

parent c791dd00
Pipeline #20560 passed with stages
in 8 minutes and 36 seconds
......@@ -55,14 +55,10 @@ secret=XXX
mb_oauth_client_id=XXX
mb_oauth_client_secret=XXX
[uwsgi]
workers=2
harakiri=120
http_timeout=90
http_connect_timeout=10
post_buffering=0
buffer_size=10240
offload_threads=1
[gunicorn]
timeout=60
workers=1
threads=1
[replication]
import_acoustid=http://data.acoustid.org/replication/acoustid-update-{seq}.xml.bz2
......
......@@ -77,7 +77,7 @@ def serialize_response(data, format, **kwargs):
def get_health_response(ctx, req, require_master=False):
# type: (ScriptContext, Request, bool) -> Response
from acoustid.uwsgi_utils import is_shutting_down
from acoustid.wsgi_utils import is_shutting_down
if require_master and ctx.config.cluster.role != 'master':
return Response('not the master server', content_type='text/plain', status=503)
if is_shutting_down(ctx.config.website.shutdown_file_path):
......
import os
import click
from typing import Optional
from acoustid.script import Script
from acoustid.uwsgi_utils import run_web_app, run_api_app
from acoustid.wsgi_utils import run_web_app, run_api_app
from acoustid.cron import run_cron
from acoustid.scripts.import_submissions import run_import
......@@ -19,26 +21,30 @@ def run():
@run.command('web')
@click.option('-c', '--config', default='acoustid.conf', envvar='ACOUSTID_CONFIG')
@click.option('-w', '--workers', type=int, envvar='ACOUSTID_WEB_WORKERS')
def run_web_cmd(config, workers):
# type: (str, int) -> None
@click.option('-w', '--workers', type=int)
@click.option('-t', '--threads', type=int)
def run_web_cmd(config, workers=None, threads=None):
# type: (str, Optional[int], Optional[int]) -> None
"""Run production uWSGI with the website."""
os.environ['ACOUSTID_CONFIG'] = config
script = Script(config)
script.setup_console_logging()
script.setup_sentry()
run_web_app(script.config, workers=workers)
run_web_app(script.config, workers=workers, threads=threads)
@run.command('api')
@click.option('-c', '--config', default='acoustid.conf', envvar='ACOUSTID_CONFIG')
@click.option('-w', '--workers', type=int, envvar='ACOUSTID_API_WORKERS')
def run_api_cmd(config, workers):
# type: (str, int) -> None
@click.option('-w', '--workers', type=int)
@click.option('-t', '--threads', type=int)
def run_api_cmd(config, workers=None, threads=None):
# type: (str, Optional[int], Optional[int]) -> None
"""Run production uWSGI with the API."""
os.environ['ACOUSTID_CONFIG'] = config
script = Script(config)
script.setup_console_logging()
script.setup_sentry()
run_api_app(script.config, workers=workers)
run_api_app(script.config, workers=workers, threads=threads)
@run.command('cron')
......
......@@ -313,6 +313,30 @@ class WebSiteConfig(BaseConfig):
read_env_item(self, 'shutdown_delay', prefix + 'SHUTDOWN_DELAY', convert=int)
class GunicornConfig(BaseConfig):
def __init__(self):
# type: () -> None
self.timeout = 90
self.workers = 1
self.threads = 1
def read_section(self, parser, section):
# type: (RawConfigParser, str) -> None
if parser.has_option(section, 'timeout'):
self.timeout = parser.getint(section, 'timeout')
if parser.has_option(section, 'workers'):
self.workers = parser.getint(section, 'workers')
if parser.has_option(section, 'threads'):
self.threads = parser.getint(section, 'workers')
def read_env(self, prefix):
# type: (str) -> None
read_env_item(self, 'timeout', prefix + 'GUNICORN_TIMEOUT', convert=int)
read_env_item(self, 'workers', prefix + 'GUNICORN_WORKERS', convert=int)
read_env_item(self, 'threads', prefix + 'GUNICORN_THREADS', convert=int)
class uWSGIConfig(BaseConfig):
def __init__(self):
......@@ -445,7 +469,7 @@ class Config(object):
self.cluster = ClusterConfig()
self.rate_limiter = RateLimiterConfig()
self.sentry = SentryConfig()
self.uwsgi = uWSGIConfig()
self.gunicorn = GunicornConfig()
def read(self, path):
# type: (str) -> None
......@@ -461,7 +485,7 @@ class Config(object):
self.cluster.read(parser, 'cluster')
self.rate_limiter.read(parser, 'rate_limiter')
self.sentry.read(parser, 'sentry')
self.uwsgi.read(parser, 'uwsgi')
self.gunicorn.read(parser, 'gunicorn')
def read_env(self, tests=False):
# type: (bool) -> None
......@@ -478,4 +502,4 @@ class Config(object):
self.cluster.read_env(prefix)
self.rate_limiter.read_env(prefix)
self.sentry.read_env(prefix)
self.uwsgi.read_env(prefix)
self.gunicorn.read_env(prefix)
......@@ -81,7 +81,7 @@ class Script(object):
if self._console_logging_configured:
return
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s - %(message)s', '%H:%M:%S'))
handler.setFormatter(logging.Formatter('[%(asctime)s] [%(process)s] [%(levelname)s] %(message)s', '%Y-%m-%d %H:%M:%S %z'))
if quiet:
handler.setLevel(logging.ERROR)
logging.getLogger().addHandler(handler)
......
# Copyright (C) 2011 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.
import os
import gzip
import sentry_sdk
from typing import Callable, List, Tuple, Any, Iterable, Optional, TYPE_CHECKING
......@@ -64,6 +65,10 @@ class Server(Script):
self.url_map = Map(url_rules, strict_slashes=False)
def __call__(self, environ, start_response):
# type: (WSGIEnvironment, StartResponse) -> Iterable[bytes]
return self.wsgi_app(environ, start_response)
def wsgi_app(self, environ, start_response):
# type: (WSGIEnvironment, StartResponse) -> Iterable[bytes]
urls = self.url_map.bind_to_environ(environ)
try:
......@@ -128,17 +133,20 @@ def add_cors_headers(app):
return wrapped_app
def make_application(config_path):
# type: (str) -> Tuple[Server, WSGIApplication]
def make_application(config_path=None):
# type: (Optional[str]) -> Server
"""Construct a WSGI application for the AcoustID server
:param config_path: path to the server configuration file
"""
if config_path is None:
config_path = os.environ.get('ACOUSTID_CONFIG', '')
assert config_path is not None
server = Server(config_path)
server.setup_sentry()
app = GzipRequestMiddleware(server) # type: WSGIApplication
app = SentryWsgiMiddleware(app)
app = replace_double_slashes(app)
app = add_cors_headers(app)
app = ProxyFix(app) # type: ignore
return server, app
server.wsgi_app = GzipRequestMiddleware(server.wsgi_app) # type: ignore
server.wsgi_app = SentryWsgiMiddleware(server.wsgi_app) # type: ignore
server.wsgi_app = replace_double_slashes(server.wsgi_app) # type: ignore
server.wsgi_app = add_cors_headers(server.wsgi_app) # type: ignore
server.wsgi_app = ProxyFix(server.wsgi_app) # type: ignore
return server
......@@ -133,7 +133,7 @@ if __name__ == "__main__":
if args.api:
app = DispatcherMiddleware(app, {
'/api': make_api_application(app.acoustid_config_filename)[1],
'/api': make_api_application(app.acoustid_config_filename),
})
run_simple(args.host, args.port, app, **run_args) # type: ignore
# Copyright (C) 2011 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.
# Simple WSGI module intended to be used by uWSGI, e.g.:
# uwsgi -w acoustid.wsgi --pythonpath ~/acoustid/ --env ACOUSTID_CONFIG=~/acoustid/acoustid.conf --http :9090
# uwsgi -w acoustid.wsgi --pythonpath ~/acoustid/ --env ACOUSTID_CONFIG=~/acoustid/acoustid.conf -M -L --socket 127.0.0.1:1717
import os
from acoustid.server import make_application
server, application = make_application(os.environ['ACOUSTID_CONFIG'])
......@@ -5,7 +5,7 @@ import signal
import io
import os
import sys
from typing import List, Callable
from typing import List, Callable, Optional
from acoustid.config import Config
if six.PY3:
import subprocess
......@@ -39,7 +39,7 @@ class ProcessWrapper(object):
logger.info('Received signal %s', sig)
if self.shutdown:
if not self.stop_immediately:
logger.info('Will stop uwsgi ASAP')
logger.info('Will stop gunicorn ASAP')
self.stop_immediately = True
else:
self.shutdown = True
......@@ -57,7 +57,7 @@ class ProcessWrapper(object):
continue
if not self.shutdown_handler_called:
logger.info('Preparing to shut down, will stop uwsgi in %s seconds', self.shutdown_delay.total_seconds())
logger.info('Preparing to shut down, will stop gunicorn in %s seconds', self.shutdown_delay.total_seconds())
if self.shutdown_handler:
self.shutdown_handler()
self.shutdown_handler_called = True
......@@ -89,7 +89,7 @@ def cleanup_shutdown_file(shutdown_file_path):
raise
def run_uwsgi(config, args):
def run_gunicorn(config, args):
# type: (Config, List[six.text_type]) -> int
cleanup_shutdown_file(config.website.shutdown_file_path)
try:
......@@ -102,61 +102,32 @@ def run_uwsgi(config, args):
cleanup_shutdown_file(config.website.shutdown_file_path)
def common_uwsgi_args(config, workers=None):
# type: (Config, int) -> List[six.text_type]
def common_gunicorn_args(config, workers=None, threads=None):
# type: (Config, Optional[int], Optional[int]) -> List[six.text_type]
args = [
os.path.join(sys.prefix, "bin", "uwsgi"),
"--die-on-term",
"--chmod-socket",
"--master",
"--disable-logging",
"--log-date",
"--workers", six.text_type(workers or config.uwsgi.workers),
"--enable-threads",
"--need-app",
os.path.join(sys.prefix, "bin", "gunicorn"),
"--workers", six.text_type(workers or config.gunicorn.workers),
"--threads", six.text_type(threads or config.gunicorn.threads),
"--limit-request-line", "8190",
]
if config.uwsgi.http_timeout:
args.extend(["--http-timeout", six.text_type(config.uwsgi.http_timeout)])
if config.uwsgi.http_connect_timeout:
args.extend(["--http-connect-timeout", six.text_type(config.uwsgi.http_connect_timeout)])
if config.uwsgi.buffer_size:
args.extend([
"--buffer-size", six.text_type(config.uwsgi.buffer_size),
"--http-buffer-size", six.text_type(config.uwsgi.buffer_size),
])
if config.uwsgi.harakiri:
args.extend([
"--harakiri", six.text_type(config.uwsgi.harakiri),
"--harakiri-verbose",
])
if config.uwsgi.offload_threads:
args.extend(["--offload-threads", six.text_type(config.uwsgi.offload_threads)])
if config.uwsgi.post_buffering:
args.extend(["--post-buffering", six.text_type(config.uwsgi.post_buffering)])
if 'PYTHONPATH' in os.environ:
args.extend(["--python-path", os.environ['PYTHONPATH']])
if hasattr(sys, 'real_prefix'):
args.extend(["--virtualenv", sys.prefix])
if config.gunicorn.timeout:
args.extend(["--timeout", six.text_type(config.gunicorn.timeout)])
return args
def run_api_app(config, workers=None):
# type: (Config, int) -> int
args = common_uwsgi_args(config, workers=workers) + [
"--http-socket", "0.0.0.0:3031",
"--module", "acoustid.wsgi",
def run_api_app(config, workers=None, threads=None):
# type: (Config, Optional[int], Optional[int]) -> int
args = common_gunicorn_args(config, workers=workers, threads=threads) + [
"--bind", "0.0.0.0:3031",
"acoustid.server:make_application()",
]
return run_uwsgi(config, args)
def run_web_app(config, workers=None):
# type: (Config, int) -> int
static_dir = os.path.join(os.path.dirname(__file__), 'web', 'static')
args = common_uwsgi_args(config, workers=workers) + [
"--http-socket", "0.0.0.0:3032",
"--module", "acoustid.web.app:make_application()",
"--static-map", "/static={}".format(static_dir),
"--static-map", "/favicon.ico={}".format(os.path.join(static_dir, 'favicon.ico')),
"--static-map", "/robots.txt={}".format(os.path.join(static_dir, 'robots.txt')),
return run_gunicorn(config, args)
def run_web_app(config, workers=None, threads=None):
# type: (Config, Optional[int], Optional[int]) -> int
args = common_gunicorn_args(config, workers=workers, threads=threads) + [
"--bind", "0.0.0.0:3032",
"acoustid.web.app:make_application()",
]
return run_uwsgi(config, args)
return run_gunicorn(config, args)
......@@ -25,3 +25,4 @@ click
ipython
attrs
gunicorn
futures
......@@ -18,6 +18,7 @@ cryptography==2.7 # via pyopenssl
decorator==4.4.0 # via ipython, traitlets
enum34==1.1.6 # via cryptography, traitlets
flask==1.1.1
futures==3.3.0
gunicorn==19.9.0
idna==2.8 # via requests
ipaddress==1.0.22 # via cryptography
......
......@@ -17,6 +17,7 @@ cryptography==2.7 # via pyopenssl
decorator==4.4.0 # via ipython, traitlets
defusedxml==0.6.0 # via python3-openid
flask==1.1.1
futures==3.1.1
gunicorn==19.9.0
idna==2.8 # via requests
ipython-genutils==0.2.0 # via traitlets
......
#!/usr/bin/env python
import os
import logging
from werkzeug.serving import run_simple
from acoustid.server import make_application
logging.basicConfig(level=logging.DEBUG)
config_path = os.path.dirname(os.path.abspath(__file__)) + '/../acoustid.conf'
server, application = make_application(config_path)
# server static files
static_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'static'))
static_files = {
'/favicon.ico': os.path.join(static_path, 'favicon.ico'),
'/static': static_path,
}
host = '0.0.0.0'
port = 5000
run_simple(host, port, application, use_reloader=True, static_files=static_files, processes=5)
......@@ -38,6 +38,21 @@ def test_gzip_request_middleware():
mw(environ, dummy_start_response)
def test_gzip_request_middleware_invalid_gzip():
# type: () -> None
def app(environ, start_response):
assert_equals(b'Hello world!', environ['wsgi.input'].read(int(environ['CONTENT_LENGTH'])))
data = b'Hello world!'
environ = {
u'HTTP_CONTENT_ENCODING': 'gzip',
u'CONTENT_LENGTH': len(data),
u'wsgi.input': BytesIO(data),
}
wsgiref.util.setup_testing_defaults(environ)
mw = GzipRequestMiddleware(app)
mw(environ, dummy_start_response)
def test_replace_double_slashes():
# type: () -> None
def app(environ, start_response):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment