config.py 15.8 KB
Newer Older
Lukáš Lalinský's avatar
Lukáš Lalinský committed
1
# Copyright (C) 2011 Lukas Lalinsky
2
# Distributed under the MIT license, see the LICENSE file for details.
Lukáš Lalinský's avatar
Lukáš Lalinský committed
3

4
import os.path
Lukáš Lalinský's avatar
Lukáš Lalinský committed
5
import logging
6
from six.moves import configparser as ConfigParser
Lukáš Lalinský's avatar
Lukáš Lalinský committed
7
from sqlalchemy import create_engine
Lukáš Lalinský's avatar
Lukáš Lalinský committed
8 9 10 11 12
from sqlalchemy.engine.url import URL

logger = logging.getLogger(__name__)


13 14 15 16 17
def str_to_bool(x):
    return x.lower() in ('1', 'on', 'true')


def read_env_item(obj, key, name, convert=None):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
18
    value = None
19 20
    if name in os.environ:
        value = os.environ[name]
Lukáš Lalinský's avatar
Lukáš Lalinský committed
21 22
        logger.info('Reading config value from environment variable %s', name)
    elif name + '_FILE' in os.environ:
23
        value = open(os.environ[name + '_FILE']).read().strip()
Lukáš Lalinský's avatar
Lukáš Lalinský committed
24
        logger.info('Reading config value from environment variable %s', name + '_FILE')
25 26 27 28 29 30
    if value is not None:
        if convert is not None:
            value = convert(value)
        setattr(obj, key, value)


Lukáš Lalinský's avatar
Lukáš Lalinský committed
31 32 33 34 35 36 37 38 39 40 41 42 43
class BaseConfig(object):

    def read(self, parser, section):
        if parser.has_section(section):
            self.read_section(parser, section)

    def read_section(self, parser, section):
        pass

    def read_env(self, prefix):
        pass


Lukáš Lalinský's avatar
Lukáš Lalinský committed
44 45 46
class DatabasesConfig(BaseConfig):

    def __init__(self):
47 48 49 50 51 52 53
        self.modes = {'local', 'replica'}
        self.names = {'main', 'import'}
        self.databases = {}
        for mode in self.modes:
            self.databases[mode] = {}
            for name in self.name:
                self.databases[mode][name] = DatabaseConfig()
Lukáš Lalinský's avatar
Lukáš Lalinský committed
54 55 56
        self.use_two_phase_commit = False

    def create_engines(self, **kwargs):
57
        mode = kwargs.pop('mode', 'local')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
58
        engines = {}
59
        for name, db_config in self.databases[mode].items():
Lukáš Lalinský's avatar
Lukáš Lalinský committed
60 61 62 63 64 65
            engines[name] = db_config.create_engine(**kwargs)
        return engines

    def read_section(self, parser, section):
        if parser.has_option(section, 'two_phase_commit'):
            self.use_two_phase_commit = parser.getboolean(section, 'two_phase_commit')
66 67 68 69
        for mode, db_configs in self.databases.items():
            for name, sub_config in db_configs.items():
                sub_section = '{}:{}{}'.format(section, name, '_' + mode if mode != 'local' else '')
                sub_config.read_section(parser, sub_section)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
70 71 72

    def read_env(self, prefix):
        read_env_item(self, 'use_two_phase_commit', prefix + 'TWO_PHASE_COMMIT', convert=str_to_bool)
73 74 75 76
        for mode, db_configs in self.databases.items():
            for name, sub_config in db_configs.items():
                sub_prefix = prefix + name.upper() + '_' + (mode.upper() + '_' if mode != 'local' else '')
                sub_config.read_env(sub_prefix)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
77 78


Lukáš Lalinský's avatar
Lukáš Lalinský committed
79
class DatabaseConfig(BaseConfig):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
80 81 82 83 84 85 86 87

    def __init__(self):
        self.user = None
        self.superuser = 'postgres'
        self.name = None
        self.host = None
        self.port = None
        self.password = None
88 89 90
        self.pool_size = None
        self.pool_recycle = None
        self.pool_pre_ping = None
Lukáš Lalinský's avatar
Lukáš Lalinský committed
91 92 93 94 95 96 97 98 99 100 101 102 103 104

    def create_url(self, superuser=False):
        kwargs = {}
        if superuser:
            kwargs['username'] = self.superuser
        else:
            kwargs['username'] = self.user
        kwargs['database'] = self.name
        if self.host is not None:
            kwargs['host'] = self.host
        if self.port is not None:
            kwargs['port'] = self.port
        if self.password is not None:
            kwargs['password'] = self.password
Lukáš Lalinský's avatar
Lukáš Lalinský committed
105
        return URL('postgresql', **kwargs)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
106

Lukáš Lalinský's avatar
Lukáš Lalinský committed
107
    def create_engine(self, superuser=False, **kwargs):
108 109 110 111 112 113
        if self.pool_size is not None and 'pool_size' not in kwargs:
            kwargs['pool_size'] = self.pool_size
        if self.pool_recycle is not None and 'pool_recycle' not in kwargs:
            kwargs['pool_recycle'] = self.pool_recycle
        if self.pool_pre_ping is not None and 'pool_pre_ping' not in kwargs:
            kwargs['pool_pre_ping'] = self.pool_pre_ping
Lukáš Lalinský's avatar
Lukáš Lalinský committed
114
        return create_engine(self.create_url(superuser=superuser), **kwargs)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
115

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
    def create_psql_args(self, superuser=False):
        args = []
        if superuser:
            args.append('-U')
            args.append(self.superuser)
        else:
            args.append('-U')
            args.append(self.user)
        if self.host is not None:
            args.append('-h')
            args.append(self.host)
        if self.port is not None:
            args.append('-p')
            args.append(str(self.port))
        args.append(self.name)
        return args

Lukáš Lalinský's avatar
Lukáš Lalinský committed
133
    def read_section(self, parser, section):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
134 135 136 137 138 139 140 141
        self.user = parser.get(section, 'user')
        self.name = parser.get(section, 'name')
        if parser.has_option(section, 'host'):
            self.host = parser.get(section, 'host')
        if parser.has_option(section, 'port'):
            self.port = parser.getint(section, 'port')
        if parser.has_option(section, 'password'):
            self.password = parser.get(section, 'password')
142
        if parser.has_option(section, 'pool_size'):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
143
            self.pool_size = parser.getint(section, 'pool_size')
144
        if parser.has_option(section, 'pool_recycle'):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
145
            self.pool_recycle = parser.getint(section, 'pool_recycle')
146
        if parser.has_option(section, 'pool_pre_ping'):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
147
            self.pool_pre_ping = parser.getboolean(section, 'pool_pre_ping')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
148

149
    def read_env(self, prefix):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
150 151 152 153 154 155 156 157
        read_env_item(self, 'name', prefix + 'NAME')
        read_env_item(self, 'host', prefix + 'HOST')
        read_env_item(self, 'port', prefix + 'PORT', convert=int)
        read_env_item(self, 'user', prefix + 'USER')
        read_env_item(self, 'password', prefix + 'PASSWORD')
        read_env_item(self, 'pool_size', prefix + 'POOL_SIZE', convert=int)
        read_env_item(self, 'pool_recycle', prefix + 'POOL_RECYCLE', convert=int)
        read_env_item(self, 'pool_pre_ping', prefix + 'POOL_PRE_PING', convert=str_to_bool)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
158

Lukáš Lalinský's avatar
Lukáš Lalinský committed
159

Lukáš Lalinský's avatar
Lukáš Lalinský committed
160
class IndexConfig(BaseConfig):
161 162 163 164 165

    def __init__(self):
        self.host = '127.0.0.1'
        self.port = 6080

Lukáš Lalinský's avatar
Lukáš Lalinský committed
166
    def read_section(self, parser, section):
167 168 169 170 171
        if parser.has_option(section, 'host'):
            self.host = parser.get(section, 'host')
        if parser.has_option(section, 'port'):
            self.port = parser.getint(section, 'port')

172 173 174
    def read_env(self, prefix):
        read_env_item(self, 'host', prefix + 'INDEX_HOST')
        read_env_item(self, 'port', prefix + 'INDEX_PORT', convert=int)
175

176

Lukáš Lalinský's avatar
Lukáš Lalinský committed
177
class RedisConfig(BaseConfig):
178 179 180 181 182

    def __init__(self):
        self.host = '127.0.0.1'
        self.port = 6379

Lukáš Lalinský's avatar
Lukáš Lalinský committed
183
    def read_section(self, parser, section):
184 185 186 187 188
        if parser.has_option(section, 'host'):
            self.host = parser.get(section, 'host')
        if parser.has_option(section, 'port'):
            self.port = parser.getint(section, 'port')

189 190 191
    def read_env(self, prefix):
        read_env_item(self, 'host', prefix + 'REDIS_HOST')
        read_env_item(self, 'port', prefix + 'REDIS_PORT', convert=int)
192

193

194 195 196 197 198 199 200 201 202 203
def get_logging_level_names():
    try:
        return logging._levelNames
    except AttributeError:
        level_names = {}
        for value, name in logging._levelToName.items():
            level_names[name] = value
        return level_names


Lukáš Lalinský's avatar
Lukáš Lalinský committed
204
class LoggingConfig(BaseConfig):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
205 206

    def __init__(self):
207
        self.levels = {}
Lukáš Lalinský's avatar
Lukáš Lalinský committed
208 209
        self.syslog = False
        self.syslog_facility = None
Lukáš Lalinský's avatar
Lukáš Lalinský committed
210

Lukáš Lalinský's avatar
Lukáš Lalinský committed
211
    def read_section(self, parser, section):
212
        level_names = get_logging_level_names()
213 214 215 216 217
        for name in parser.options(section):
            if name == 'level':
                self.levels[''] = level_names[parser.get(section, name)]
            elif name.startswith('level.'):
                self.levels[name.split('.', 1)[1]] = level_names[parser.get(section, name)]
Lukáš Lalinský's avatar
Lukáš Lalinský committed
218 219 220 221
        if parser.has_option(section, 'syslog'):
            self.syslog = parser.getboolean(section, 'syslog')
        if parser.has_option(section, 'syslog_facility'):
            self.syslog_facility = parser.get(section, 'syslog_facility')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
222

223
    def read_env(self, prefix):
224 225
        pass  # XXX

Lukáš Lalinský's avatar
Lukáš Lalinský committed
226

Lukáš Lalinský's avatar
Lukáš Lalinský committed
227
class WebSiteConfig(BaseConfig):
228 229

    def __init__(self):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
230
        self.debug = False
231
        self.secret = None
232 233
        self.mb_oauth_client_id = None
        self.mb_oauth_client_secret = None
Lukáš Lalinský's avatar
Lukáš Lalinský committed
234 235
        self.google_oauth_client_id = None
        self.google_oauth_client_secret = None
Lukáš Lalinský's avatar
Lukáš Lalinský committed
236
        self.maintenance = False
Lukáš Lalinský's avatar
Lukáš Lalinský committed
237
        self.shutdown_delay = 0
238
        self.shutdown_file_path = '/tmp/acoustid-server-shutdown.txt'
239

Lukáš Lalinský's avatar
Lukáš Lalinský committed
240
    def read_section(self, parser, section):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
241 242
        if parser.has_option(section, 'debug'):
            self.debug = parser.getboolean(section, 'debug')
243
        self.secret = parser.get(section, 'secret')
244 245 246 247
        if parser.has_option(section, 'mb_oauth_client_id'):
            self.mb_oauth_client_id = parser.get(section, 'mb_oauth_client_id')
        if parser.has_option(section, 'mb_oauth_client_secret'):
            self.mb_oauth_client_secret = parser.get(section, 'mb_oauth_client_secret')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
248 249 250 251
        if parser.has_option(section, 'google_oauth_client_id'):
            self.google_oauth_client_id = parser.get(section, 'google_oauth_client_id')
        if parser.has_option(section, 'google_oauth_client_secret'):
            self.google_oauth_client_secret = parser.get(section, 'google_oauth_client_secret')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
252 253
        if parser.has_option(section, 'maintenance'):
            self.maintenance = parser.getboolean(section, 'maintenance')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
254 255
        if parser.has_option(section, 'shutdown_delay'):
            self.shutdown_delay = parser.getint(section, 'shutdown_delay')
256

257 258 259 260 261 262 263
    def read_env(self, prefix):
        read_env_item(self, 'debug', prefix + 'DEBUG', convert=str_to_bool)
        read_env_item(self, 'maintenance', prefix + 'MAINTENANCE', convert=str_to_bool)
        read_env_item(self, 'mb_oauth_client_id', prefix + 'MB_OAUTH_CLIENT_ID')
        read_env_item(self, 'mb_oauth_client_secret', prefix + 'MB_OAUTH_CLIENT_SECRET')
        read_env_item(self, 'google_oauth_client_id', prefix + 'GOOGLE_OAUTH_CLIENT_ID')
        read_env_item(self, 'google_oauth_client_secret', prefix + 'GOOGLE_OAUTH_CLIENT_SECRET')
264
        read_env_item(self, 'shutdown_delay', prefix + 'SHUTDOWN_DELAY', convert=int)
265

266

Lukáš Lalinský's avatar
Lukáš Lalinský committed
267
class uWSGIConfig(BaseConfig):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
268 269

    def __init__(self):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
270 271 272
        self.harakiri = 120
        self.http_timeout = 90
        self.http_connect_timeout = 10
Lukáš Lalinský's avatar
Lukáš Lalinský committed
273 274 275 276 277
        self.workers = 2
        self.post_buffering = 0
        self.buffer_size = 10240
        self.offload_threads = 1

Lukáš Lalinský's avatar
Lukáš Lalinský committed
278
    def read_section(self, parser, section):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
279 280
        if parser.has_option(section, 'harakiri'):
            self.harakiri = parser.getint(section, 'harakiri')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
281 282 283 284
        if parser.has_option(section, 'http_timeout'):
            self.http_timeout = parser.getint(section, 'http_timeout')
        if parser.has_option(section, 'http_connect_timeout'):
            self.http_connect_timeout = parser.getint(section, 'http_connect_timeout')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
285 286 287 288 289 290 291 292 293 294 295
        if parser.has_option(section, 'workers'):
            self.workers = parser.getint(section, 'workers')
        if parser.has_option(section, 'post_buffering'):
            self.post_buffering = parser.getint(section, 'post_buffering')
        if parser.has_option(section, 'buffer_size'):
            self.buffer_size = parser.getint(section, 'buffer_size')
        if parser.has_option(section, 'offload_threads'):
            self.offload_threads = parser.getint(section, 'offload_threads')

    def read_env(self, prefix):
        read_env_item(self, 'harakiri', prefix + 'UWSGI_HARAKIRI', convert=int)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
296 297
        read_env_item(self, 'http_timeout', prefix + 'UWSGI_HTTP_TIMEOUT', convert=int)
        read_env_item(self, 'http_connect_timeout', prefix + 'UWSGI_CONNECT_TIMEOUT', convert=int)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
298 299 300 301 302 303
        read_env_item(self, 'workers', prefix + 'UWSGI_WORKERS', convert=int)
        read_env_item(self, 'post_buffering', prefix + 'UWSGI_POST_BUFFERING', convert=int)
        read_env_item(self, 'buffer_size', prefix + 'UWSGI_BUFFER_SIZE', convert=int)
        read_env_item(self, 'offload_threads', prefix + 'UWSGI_OFFLOAD_THREADS', convert=int)


Lukáš Lalinský's avatar
Lukáš Lalinský committed
304
class SentryConfig(BaseConfig):
305 306 307 308

    def __init__(self):
        self.web_dsn = ''
        self.api_dsn = ''
Lukáš Lalinský's avatar
Lukáš Lalinský committed
309
        self.script_dsn = ''
310

Lukáš Lalinský's avatar
Lukáš Lalinský committed
311
    def read_section(self, parser, section):
312 313 314
        if parser.has_option(section, 'web_dsn'):
            self.web_dsn = parser.get(section, 'web_dsn')
        if parser.has_option(section, 'api_dsn'):
315
            self.api_dsn = parser.get(section, 'api_dsn')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
316
        if parser.has_option(section, 'script_dsn'):
317
            self.script_dsn = parser.get(section, 'script_dsn')
318 319 320

    def read_env(self, prefix):
        read_env_item(self, 'web_dsn', prefix + 'SENTRY_WEB_DSN')
321
        read_env_item(self, 'api_dsn', prefix + 'SENTRY_API_DSN')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
322
        read_env_item(self, 'script_dsn', prefix + 'SENTRY_SCRIPT_DSN')
323 324


Lukáš Lalinský's avatar
Lukáš Lalinský committed
325
class ReplicationConfig(BaseConfig):
326

adam's avatar
adam committed
327 328 329 330
    def __init__(self):
        self.import_acoustid = None
        self.import_acoustid_musicbrainz = None

Lukáš Lalinský's avatar
Lukáš Lalinský committed
331
    def read_section(self, parser, section):
adam's avatar
adam committed
332 333 334 335 336
        if parser.has_option(section, 'import_acoustid'):
            self.import_acoustid = parser.get(section, 'import_acoustid')
        if parser.has_option(section, 'import_acoustid_musicbrainz'):
            self.import_acoustid_musicbrainz = parser.get(section, 'import_acoustid_musicbrainz')

337
    def read_env(self, prefix):
338 339
        pass  # XXX

adam's avatar
adam committed
340

Lukáš Lalinský's avatar
Lukáš Lalinský committed
341
class ClusterConfig(BaseConfig):
342 343 344 345 346 347

    def __init__(self):
        self.role = 'master'
        self.base_master_url = None
        self.secret = None

Lukáš Lalinský's avatar
Lukáš Lalinský committed
348
    def read_section(self, parser, section):
349 350 351 352 353 354 355
        if parser.has_option(section, 'role'):
            self.role = parser.get(section, 'role')
        if parser.has_option(section, 'base_master_url'):
            self.base_master_url = parser.get(section, 'base_master_url')
        if parser.has_option(section, 'secret'):
            self.secret = parser.get(section, 'secret')

356 357 358 359
    def read_env(self, prefix):
        read_env_item(self, 'role', prefix + 'CLUSTER_ROLE')
        read_env_item(self, 'base_master_url', prefix + 'CLUSTER_BASE_MASTER_URL')
        read_env_item(self, 'secret', prefix + 'CLUSTER_SECRET')
360

361

Lukáš Lalinský's avatar
Lukáš Lalinský committed
362
class RateLimiterConfig(BaseConfig):
363 364 365 366 367

    def __init__(self):
        self.ips = {}
        self.applications = {}

Lukáš Lalinský's avatar
Lukáš Lalinský committed
368
    def read_section(self, parser, section):
369 370 371 372 373 374
        for name in parser.options(section):
            if name.startswith('ip.'):
                self.ips[name.split('.', 1)[1]] = parser.getfloat(section, name)
            elif name.startswith('application.'):
                self.applications[int(name.split('.', 1)[1])] = parser.getfloat(section, name)

375
    def read_env(self, prefix):
376 377
        pass  # XXX

378

Lukáš Lalinský's avatar
Lukáš Lalinský committed
379 380
class Config(object):

381
    def __init__(self):
Lukáš Lalinský's avatar
Lukáš Lalinský committed
382
        self.databases = DatabasesConfig()
383 384 385 386 387 388 389
        self.logging = LoggingConfig()
        self.website = WebSiteConfig()
        self.index = IndexConfig()
        self.redis = RedisConfig()
        self.replication = ReplicationConfig()
        self.cluster = ClusterConfig()
        self.rate_limiter = RateLimiterConfig()
390
        self.sentry = SentryConfig()
Lukáš Lalinský's avatar
Lukáš Lalinský committed
391
        self.uwsgi = uWSGIConfig()
392 393

    def read(self, path):
394
        logger.info("Loading configuration file %s", path)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
395 396
        parser = ConfigParser.RawConfigParser()
        parser.read(path)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
397
        self.databases.read(parser, 'database')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
398
        self.logging.read(parser, 'logging')
399
        self.website.read(parser, 'website')
400
        self.index.read(parser, 'index')
401
        self.redis.read(parser, 'redis')
adam's avatar
adam committed
402
        self.replication.read(parser, 'replication')
403
        self.cluster.read(parser, 'cluster')
404
        self.rate_limiter.read(parser, 'rate_limiter')
405
        self.sentry.read(parser, 'sentry')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
406
        self.uwsgi.read(parser, 'uwsgi')
Lukáš Lalinský's avatar
Lukáš Lalinský committed
407

408 409 410 411 412
    def read_env(self, tests=False):
        if tests:
            prefix = 'ACOUSTID_TEST_'
        else:
            prefix = 'ACOUSTID_'
413
        self.databases.read_env(prefix + 'DB_')
414 415 416 417 418 419 420
        self.logging.read_env(prefix)
        self.website.read_env(prefix)
        self.index.read_env(prefix)
        self.redis.read_env(prefix)
        self.replication.read_env(prefix)
        self.cluster.read_env(prefix)
        self.rate_limiter.read_env(prefix)
421
        self.sentry.read_env(prefix)
Lukáš Lalinský's avatar
Lukáš Lalinský committed
422
        self.uwsgi.read_env(prefix)