Commit c621be23 authored by Lukáš Lalinský's avatar Lukáš Lalinský

Add submission result

parent 9fb5ab9e
Pipeline #20474 passed with stages
in 3 minutes
......@@ -8,6 +8,16 @@ from acoustid import tables as schema
logger = logging.getLogger(__name__)
def get_foreignid(conn, id):
src = schema.foreignid.join(schema.foreignid_vendor, schema.foreignid.c.vendor_id == schema.foreignid_vendor.c.id)
query = sql.select([
schema.foreignid_vendor.c.name.label('namespace'),
schema.foreignid.c.name.label('id'),
], schema.foreignid.c.id == id, from_obj=src)
row = conn.execute(query).first()
return row['namespace'] + ':' + row['id']
def find_or_insert_foreignid_vendor(conn, name):
with conn.begin():
query = sql.select([schema.foreignid_vendor.c.id], schema.foreignid_vendor.c.name == name)
......
......@@ -8,6 +8,15 @@ from acoustid import tables as schema
logger = logging.getLogger(__name__)
def get_source(conn, id):
query = sql.select([
schema.source.c.account_id,
schema.source.c.application_id,
schema.source.c.version,
], schema.source.c.id == id)
return conn.execute(query).first()
def find_or_insert_source(conn, application_id, account_id, version=None):
"""
Find a source in the database, create it if it doesn't exist yet.
......
......@@ -5,11 +5,12 @@ import logging
from sqlalchemy import sql
from acoustid import tables as schema, const
from acoustid.data.fingerprint import insert_fingerprint, inc_fingerprint_submission_count, FingerprintSearcher
from acoustid.data.musicbrainz import resolve_mbid_redirect
from acoustid.data.track import (
insert_track, insert_mbid, insert_puid, merge_tracks, insert_track_meta,
can_add_fp_to_track, can_merge_tracks, insert_track_foreignid,
)
from acoustid.data.source import get_source
from acoustid.data.foreignid import get_foreignid
logger = logging.getLogger(__name__)
......@@ -35,6 +36,27 @@ def insert_submission(conn, data):
return id
def insert_submission_result(conn, data):
"""
Insert a new submission result into the database
"""
with conn.begin():
insert_stmt = schema.submission_result.insert().values({
'submission_id': data['submission_id'],
'created': data['created'],
'account_id': data['account_id'],
'application_id': data['application_id'],
'application_version': data.get('application_version'),
'fingerprint_id': data['fingerprint_id'],
'track_id': data['track_id'],
'mbid': data.get('mbid'),
'puid': data.get('puid'),
'meta_id': data.get('meta_id'),
'foreignid': data.get('foreignid'),
})
conn.execute(insert_stmt)
def import_submission(conn, submission, index=None):
"""
Import the given submission into the main fingerprint database
......@@ -43,22 +65,28 @@ def import_submission(conn, submission, index=None):
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
conn.execute(update_stmt.values(handled=True))
mbids = []
if submission['mbid']:
mbids.append(resolve_mbid_redirect(conn, submission['mbid']))
logger.info("Importing submission %d with MBIDs %s",
submission['id'], ', '.join(mbids))
logger.info("Importing submission %d with MBIDs %s", submission['id'], submission['mbid'])
num_unique_items = len(set(submission['fingerprint']))
if num_unique_items < const.FINGERPRINT_MIN_UNIQUE_ITEMS:
logger.info("Skipping, has only %d unique items", num_unique_items)
return
num_query_items = conn.execute("SELECT icount(acoustid_extract_query(%(fp)s))", dict(fp=submission['fingerprint']))
if not num_query_items:
logger.info("Skipping, no data to index")
return
searcher = FingerprintSearcher(conn, index, fast=False)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(submission['fingerprint'], submission['length'])
source = get_source(conn, submission['source_id'])
submission_result = {
'submission_id': submission['id'],
'created': submission['created'],
'account_id': source['account_id'],
'application_id': source['application_id'],
'application_version': source['version'],
}
fingerprint = {
'id': None,
'track_id': None,
......@@ -67,6 +95,10 @@ def import_submission(conn, submission, index=None):
'bitrate': submission['bitrate'],
'format_id': submission['format_id'],
}
searcher = FingerprintSearcher(conn, index, fast=False)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(submission['fingerprint'], submission['length'])
if matches:
all_track_ids = set()
possible_track_ids = set()
......@@ -88,20 +120,37 @@ def import_submission(conn, submission, index=None):
group.remove(fingerprint['track_id'])
merge_tracks(conn, fingerprint['track_id'], list(group))
break
if not fingerprint['track_id']:
fingerprint['track_id'] = insert_track(conn)
submission_result['track_id'] = fingerprint['track_id']
if not fingerprint['id']:
fingerprint['id'] = insert_fingerprint(conn, fingerprint, submission['id'], submission['source_id'])
else:
inc_fingerprint_submission_count(conn, fingerprint['id'], submission['id'], submission['source_id'])
for mbid in mbids:
insert_mbid(conn, fingerprint['track_id'], mbid, submission['id'], submission['source_id'])
submission_result['fingerprint_id'] = fingerprint['id']
if submission['mbid'] and submission['mbid'] != '00000000-0000-0000-0000-000000000000':
insert_mbid(conn, fingerprint['track_id'], submission['mbid'], submission['id'], submission['source_id'])
submission_result['mbid'] = submission['mbid']
if submission['puid'] and submission['puid'] != '00000000-0000-0000-0000-000000000000':
insert_puid(conn, fingerprint['track_id'], submission['puid'], submission['id'], submission['source_id'])
submission_result['puid'] = submission['puid']
if submission['meta_id']:
insert_track_meta(conn, fingerprint['track_id'], submission['meta_id'], submission['id'], submission['source_id'])
submission_result['meta_id'] = submission['meta_id']
if submission['foreignid_id']:
insert_track_foreignid(conn, fingerprint['track_id'], submission['foreignid_id'], submission['id'], submission['source_id'])
submission_result['foreignid'] = get_foreignid(conn, submission['foreignid_id'])
insert_submission_result(conn, submission_result)
return fingerprint
......
......@@ -55,6 +55,14 @@ class Source(Base):
account = relationship('Account')
class Submission(Base):
__table__ = tables.submission
class SubmissionResult(Base):
__table__ = tables.submission_result
class Track(Base):
__table__ = tables.track
......
......@@ -107,6 +107,26 @@ submission = Table('submission', metadata,
Index('submission_idx_handled', submission.c.id, postgresql_where=submission.c.handled == False) # noqa: E712
submission_result = Table('submission_result', metadata,
Column('submission_id', Integer, primary_key=True, autoincrement=False),
Column('created', DateTime(timezone=True), nullable=False),
# source
Column('account_id', Integer, nullable=False),
Column('application_id', Integer, nullable=False),
Column('application_version', String),
# fingerprint
Column('fingerprint_id', Integer, nullable=False),
Column('track_id', Integer, nullable=False),
# metadata
Column('meta_id', Integer),
Column('mbid', UUID),
Column('puid', UUID),
Column('foreignid', String),
)
stats = Table('stats', metadata,
Column('id', Integer, primary_key=True),
Column('name', String, nullable=False),
......
"""remove recording_acoustid
Revision ID: 1550d60e9261
Revises: f99ba0e4ed97
Create Date: 2019-09-14 15:28:45.868715
"""
# revision identifiers, used by Alembic.
revision = '1550d60e9261'
down_revision = 'f99ba0e4ed97'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
op.drop_index('recording_acoustid_idx_acoustid', table_name='recording_acoustid')
op.drop_index('recording_acoustid_idx_uniq', table_name='recording_acoustid')
op.drop_table('recording_acoustid')
def downgrade():
op.create_table('recording_acoustid',
sa.Column('id', sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column('acoustid', postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column('recording', postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column('disabled', sa.BOOLEAN(), server_default=sa.text(u'false'), autoincrement=False, nullable=False),
sa.Column('created', postgresql.TIMESTAMP(timezone=True), server_default=sa.text(u'CURRENT_TIMESTAMP'), autoincrement=False, nullable=False),
sa.Column('updated', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True),
sa.PrimaryKeyConstraint('id', name=u'recording_acoustid_pkey')
)
op.create_index('recording_acoustid_idx_uniq', 'recording_acoustid', ['recording', 'acoustid'], unique=True)
op.create_index('recording_acoustid_idx_acoustid', 'recording_acoustid', ['acoustid'], unique=False)
"""add submission_result
Revision ID: 64eb873263b6
Revises: 1550d60e9261
Create Date: 2019-09-14 15:45:52.593164
"""
# revision identifiers, used by Alembic.
revision = '64eb873263b6'
down_revision = '1550d60e9261'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
op.create_table('submission_result',
sa.Column('submission_id', sa.Integer(), autoincrement=False, nullable=False),
sa.Column('created', sa.DateTime(timezone=True), nullable=False),
sa.Column('account_id', sa.Integer(), nullable=False),
sa.Column('application_id', sa.Integer(), nullable=False),
sa.Column('application_version', sa.String(), nullable=True),
sa.Column('fingerprint_id', sa.Integer(), nullable=False),
sa.Column('track_id', sa.Integer(), nullable=False),
sa.Column('meta_id', sa.Integer(), nullable=True),
sa.Column('mbid', postgresql.UUID(), nullable=True),
sa.Column('puid', postgresql.UUID(), nullable=True),
sa.Column('foreignid', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('submission_id', name=op.f('submission_result_pkey'))
)
def downgrade():
op.drop_table('submission_result')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment