Commit 3bfb3a9b authored by Lukáš Lalinský's avatar Lukáš Lalinský

Make submission more self-sustained

parent 869bb636
Pipeline #20639 failed with stages
in 4 seconds
......@@ -765,13 +765,19 @@ class SubmitHandler(APIHandler):
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
# 'format': p['format'] or None,
'format_id': p.get('format_id'),
'source_id': source_id
# 'account_id': params.account_id,
# 'application_id': params.application_id,
# 'application_version': params.application_version,
'source_id': source_id,
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.values()):
# values['meta'] = meta_values
values['meta_id'] = insert_meta(fingerprint_db, meta_values)
if p['foreignid']:
# values['foreignid'] = p['foreignid']
values['foreignid_id'] = find_or_insert_foreignid(fingerprint_db, p['foreignid'])
id = insert_submission(ingest_db, values)
ids.add(id)
......
......@@ -2,6 +2,8 @@
# Distributed under the MIT license, see the LICENSE file for details.
import logging
# import datetime
# import pytz
from typing import Dict, Any, Optional, Set, List, Iterable
from sqlalchemy import sql
from acoustid import tables as schema, const
......@@ -18,46 +20,29 @@ from acoustid.data.foreignid import get_foreignid
logger = logging.getLogger(__name__)
def insert_submission(ingest_db, data):
def insert_submission(ingest_db, values):
# type: (IngestDB, Dict[str, Any]) -> int
"""
Insert a new submission into the database
"""
insert_stmt = schema.submission.insert().values({
'fingerprint': data['fingerprint'],
'length': data['length'],
'bitrate': data.get('bitrate'),
'mbid': data.get('mbid'),
'puid': data.get('puid'),
'source_id': data.get('source_id'),
'format_id': data.get('format_id'),
'meta_id': data.get('meta_id'),
'foreignid_id': data.get('foreignid_id'),
})
values = dict((k, v) for (k, v) in values.items() if v is not None)
insert_stmt = schema.submission.insert().values(values)
submission_id = ingest_db.execute(insert_stmt).inserted_primary_key[0]
logger.debug("Inserted submission %r with data %r", submission_id, data)
logger.debug("Inserted submission %r with data %r", submission_id, values)
return submission_id
def insert_submission_result(ingest_db, data):
# type: (IngestDB, Dict[str, Any]) -> None
def insert_submission_result(ingest_db, values):
# type: (IngestDB, Dict[str, Any]) -> int
"""
Insert a new submission result into the database
"""
insert_stmt = schema.submission_result.insert().values({
'submission_id': data['submission_id'],
'created': data['created'],
'account_id': data['account_id'],
'application_id': data['application_id'],
'application_version': data.get('application_version'),
'fingerprint_id': data['fingerprint_id'],
'track_id': data['track_id'],
'mbid': data.get('mbid'),
'puid': data.get('puid'),
'meta_id': data.get('meta_id'),
'foreignid': data.get('foreignid'),
})
values = dict((k, v) for (k, v) in values.items() if v is not None)
insert_stmt = schema.submission_result.insert().values(values)
ingest_db.execute(insert_stmt)
submission_id = int(values['submission_id'])
logger.debug("Inserted submission_result %r with data %r", submission_id, values)
return submission_id
def import_submission(ingest_db, app_db, fingerprint_db, index_pool, submission):
......@@ -65,9 +50,13 @@ def import_submission(ingest_db, app_db, fingerprint_db, index_pool, submission)
"""
Import the given submission into the main fingerprint database
"""
# handled_at = datetime.datetime.now(pytz.utc)
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
ingest_db.execute(update_stmt.values(handled=True))
# ingest_db.execute(update_stmt.values(handled=True, handled_at=handled_at))
logger.info("Importing submission %d with MBIDs %s", submission['id'], submission['mbid'])
num_unique_items = len(set(submission['fingerprint']))
......@@ -88,6 +77,7 @@ def import_submission(ingest_db, app_db, fingerprint_db, index_pool, submission)
submission_result = {
'submission_id': submission['id'],
'created': submission['created'],
# 'handled_at': handled_at,
'account_id': source['account_id'],
'application_id': source['application_id'],
'application_version': source['version'],
......
......@@ -5,7 +5,7 @@ from sqlalchemy import (
Integer, String, DateTime, Boolean, Date, Text, SmallInteger,
DDL, sql,
)
from sqlalchemy.dialects.postgresql import ARRAY, UUID, INET
from sqlalchemy.dialects.postgresql import ARRAY, UUID, INET, JSONB
metadata = MetaData(naming_convention={
'fk': '%(table_name)s_fk_%(column_0_name)s',
......@@ -94,17 +94,33 @@ source = Table('source', metadata,
submission = Table('submission', metadata,
Column('id', Integer, primary_key=True),
# status
Column('created', DateTime(timezone=True), server_default=sql.func.current_timestamp(), nullable=False),
Column('handled_at', DateTime(timezone=True)),
Column('handled', Boolean, default=False, server_default=sql.false()),
# source
Column('account_id', Integer, nullable=True),
Column('application_id', Integer, nullable=True),
Column('application_version', String),
Column('source_id', Integer, nullable=True), # XXX deprecated
# fingerprint
Column('fingerprint', ARRAY(Integer), nullable=False),
Column('length', SmallInteger, CheckConstraint('length>0'), nullable=False),
Column('bitrate', SmallInteger, CheckConstraint('bitrate>0')),
Column('format_id', Integer),
Column('created', DateTime(timezone=True), server_default=sql.func.current_timestamp(), nullable=False),
Column('source_id', Integer, nullable=False),
Column('format', String),
Column('format_id', Integer), # XXX deprecated
# metadata
Column('meta', JSONB),
Column('meta_id', Integer), # XXX deprecated
Column('mbid', UUID),
Column('handled', Boolean, default=False, server_default=sql.false()),
Column('puid', UUID),
Column('meta_id', Integer),
Column('foreignid_id', Integer),
Column('foreignid', String),
Column('foreignid_id', Integer), # XXX deprecated
info={'bind_key': 'ingest'},
)
......@@ -112,7 +128,10 @@ Index('submission_idx_handled', submission.c.id, postgresql_where=submission.c.h
submission_result = Table('submission_result', metadata,
Column('submission_id', Integer, primary_key=True, autoincrement=False),
# status
Column('created', DateTime(timezone=True), nullable=False),
Column('handled_at', DateTime(timezone=True), nullable=True),
# source
Column('account_id', Integer, nullable=False),
......
"""more fields in submission
Revision ID: 661fce6a2f64
Revises: 0c79593066ed
Create Date: 2019-11-14 12:02:55.010214
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '661fce6a2f64'
down_revision = '0c79593066ed'
branch_labels = None
depends_on = None
def upgrade(engine_name):
globals()["upgrade_%s" % engine_name]()
def downgrade(engine_name):
globals()["downgrade_%s" % engine_name]()
def upgrade_app():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def downgrade_app():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def upgrade_ingest():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('submission', sa.Column('account_id', sa.Integer(), nullable=True))
op.add_column('submission', sa.Column('application_id', sa.Integer(), nullable=True))
op.add_column('submission', sa.Column('application_version', sa.String(), nullable=True))
op.add_column('submission', sa.Column('foreignid', sa.String(), nullable=True))
op.add_column('submission', sa.Column('format', sa.String(), nullable=True))
op.add_column('submission', sa.Column('handled_at', sa.DateTime(timezone=True), nullable=True))
op.add_column('submission', sa.Column('meta', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
op.alter_column('submission', 'source_id',
existing_type=sa.INTEGER(),
nullable=True)
op.add_column('submission_result', sa.Column('handled_at', sa.DateTime(timezone=True), nullable=True))
# ### end Alembic commands ###
def downgrade_ingest():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('submission_result', 'handled_at')
op.alter_column('submission', 'source_id',
existing_type=sa.INTEGER(),
nullable=False)
op.drop_column('submission', 'meta')
op.drop_column('submission', 'handled_at')
op.drop_column('submission', 'format')
op.drop_column('submission', 'foreignid')
op.drop_column('submission', 'application_version')
op.drop_column('submission', 'application_id')
op.drop_column('submission', 'account_id')
# ### end Alembic commands ###
def upgrade_fingerprint():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
def downgrade_fingerprint():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###
......@@ -26,3 +26,4 @@ ipython
attrs
gunicorn
futures
pytz
......@@ -21,4 +21,4 @@ six==1.12.0 # via grpcio, protobuf
typing==3.7.4.1 # via flake8
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.4.0 # via protobuf
# setuptools==41.6.0 # via protobuf
......@@ -21,4 +21,4 @@ typed-ast==1.4.0 # via mypy
typing-extensions==3.7.4 # via mypy, sqlalchemy-stubs
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.4.0 # via protobuf
# setuptools==41.6.0 # via protobuf
......@@ -44,6 +44,7 @@ pyopenssl==19.0.0
python-dateutil==2.8.0 # via alembic
python-editor==1.0.4 # via alembic
python-openid==2.2.5 ; python_version < "3"
pytz==2019.3
rauth==0.7.3
redis==3.3.8
requests==2.22.0
......@@ -62,4 +63,4 @@ wcwidth==0.1.7 # via prompt-toolkit
werkzeug==0.16.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.4.0 # via ipython, markdown
# setuptools==41.6.0 # via ipython, markdown
......@@ -43,6 +43,7 @@ pyopenssl==19.0.0
python-dateutil==2.8.0 # via alembic
python-editor==1.0.4 # via alembic
python3-openid==3.1.0 ; python_version >= "3"
pytz==2019.3
rauth==0.7.3
redis==3.3.8
requests==2.22.0
......@@ -58,4 +59,4 @@ wcwidth==0.1.7 # via prompt-toolkit
werkzeug==0.16.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.4.0 # via ipython, markdown
# setuptools==41.6.0 # via ipython, markdown
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment