Commit 0b5d6edc authored by Lukáš Lalinský's avatar Lukáš Lalinský

Move fingerprint<->meta mapping to a separate table

parent 605f0e61
......@@ -60,7 +60,6 @@ def insert_fingerprint(conn, data, submission_id=None, source_id=None):
'bitrate': data.get('bitrate'),
'format_id': data.get('format_id'),
'track_id': data['track_id'],
'meta_id': data.get('meta_id'),
'submission_count': 1,
})
id = conn.execute(insert_stmt).inserted_primary_key[0]
......
......@@ -6,7 +6,7 @@ from sqlalchemy import sql
from acoustid import tables as schema
from acoustid.data.fingerprint import lookup_fingerprint, insert_fingerprint, inc_fingerprint_submission_count
from acoustid.data.musicbrainz import find_puid_mbids, resolve_mbid_redirect
from acoustid.data.track import insert_track, insert_mbid, insert_puid, merge_tracks
from acoustid.data.track import insert_track, insert_mbid, insert_puid, merge_tracks, insert_track_meta
logger = logging.getLogger(__name__)
TRACK_MERGE_TRESHOLD = 0.7
......@@ -66,7 +66,6 @@ def import_submission(conn, submission):
'length': submission['length'],
'bitrate': submission['bitrate'],
'format_id': submission['format_id'],
'meta_id': submission['meta_id'],
}
if matches:
match = matches[0]
......@@ -97,6 +96,8 @@ def import_submission(conn, submission):
insert_mbid(conn, fingerprint['track_id'], mbid, submission['id'], submission['source_id'])
if submission['puid']:
insert_puid(conn, fingerprint['track_id'], submission['puid'], submission['id'], submission['source_id'])
if submission['meta_id']:
insert_track_meta(conn, fingerprint['track_id'], submission['meta_id'], submission['id'], submission['source_id'])
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
conn.execute(update_stmt.values(handled=True))
......
......@@ -85,9 +85,10 @@ def merge_missing_mbids(conn):
merge_mbids(conn, new_mbid, old_mbids)
def _merge_tracks_gids(conn, name, target_id, source_ids):
def _merge_tracks_gids(conn, name_with_id, target_id, source_ids):
name = name_with_id.replace('_id', '')
tab = schema.metadata.tables['track_%s' % name]
col = tab.columns[name]
col = tab.columns[name_with_id]
tab_src = schema.metadata.tables['track_%s_source' % name]
col_src = tab_src.columns['track_%s_id' % name]
query = sql.select(
......@@ -125,6 +126,7 @@ def merge_tracks(conn, target_id, source_ids):
with conn.begin():
_merge_tracks_gids(conn, 'mbid', target_id, source_ids)
_merge_tracks_gids(conn, 'puid', target_id, source_ids)
_merge_tracks_gids(conn, 'meta_id', target_id, source_ids)
# XXX don't move duplicate fingerprints
update_stmt = schema.fingerprint.update().where(
schema.fingerprint.c.track_id.in_(source_ids))
......@@ -159,7 +161,7 @@ def _insert_gid(conn, tab, tab_src, col, name, track_id, gid, submission_id=None
id = conn.execute(insert_stmt).inserted_primary_key[0]
logger.debug("Added %s %s to track %d", name.upper(), gid, track_id)
insert_stmt = tab_src.insert().values({
'track_%s_id' % name: id,
'track_%s_id' % name.replace('_id', ''): id,
'submission_id': submission_id,
'source_id': source_id,
})
......@@ -177,6 +179,11 @@ def insert_puid(conn, track_id, puid, submission_id=None, source_id=None):
schema.track_puid.c.puid, 'puid', track_id, puid, submission_id, source_id)
def insert_track_meta(conn, track_id, meta_id, submission_id=None, source_id=None):
return _insert_gid(conn, schema.track_meta, schema.track_meta_source,
schema.track_meta.c.meta_id, 'meta_id', track_id, meta_id, submission_id, source_id)
def get_track_fingerprint_matrix(conn, track_id):
query = """
SELECT
......
......@@ -90,7 +90,6 @@ fingerprint = Table('fingerprint', metadata,
Column('bitrate', Integer),
Column('format_id', Integer, ForeignKey('format.id')),
Column('track_id', Integer, ForeignKey('track.id')),
Column('meta_id', Integer, ForeignKey('meta.id')),
Column('submission_count', Integer),
)
......@@ -131,6 +130,21 @@ track_puid_source = Table('track_puid_source', metadata,
Column('source_id', Integer, ForeignKey('source.id')),
)
track_meta = Table('track_meta', metadata,
Column('id', Integer, primary_key=True),
Column('track_id', Integer, ForeignKey('track.id')),
Column('meta_id', Integer, ForeignKey('track.id')),
Column('created', DateTime),
Column('submission_count', Integer),
)
track_meta_source = Table('track_meta_source', metadata,
Column('id', Integer, primary_key=True),
Column('track_meta_id', Integer, ForeignKey('track_meta.id')),
Column('submission_id', Integer, ForeignKey('submission.id')),
Column('source_id', Integer, ForeignKey('source.id')),
)
mb_artist = Table('s_artist', metadata,
Column('id', Integer, primary_key=True),
Column('name', String),
......
......@@ -66,6 +66,26 @@ ALTER TABLE track_puid_source ADD CONSTRAINT track_puid_source_fk_source_id
FOREIGN KEY (source_id)
REFERENCES source (id);
ALTER TABLE track_meta ADD CONSTRAINT track_meta_fk_track_id
FOREIGN KEY (track_id)
REFERENCES track (id);
ALTER TABLE track_meta ADD CONSTRAINT track_meta_fk_meta_id
FOREIGN KEY (meta_id)
REFERENCES meta (id);
ALTER TABLE track_meta_source ADD CONSTRAINT track_meta_source_fk_track_meta_id
FOREIGN KEY (track_meta_id)
REFERENCES track_meta (id);
ALTER TABLE track_meta_source ADD CONSTRAINT track_meta_source_fk_submission_id
FOREIGN KEY (submission_id)
REFERENCES submission (id);
ALTER TABLE track_meta_source ADD CONSTRAINT track_meta_source_fk_source_id
FOREIGN KEY (source_id)
REFERENCES source (id);
ALTER TABLE submission ADD CONSTRAINT submission_fk_source_id
FOREIGN KEY (source_id)
REFERENCES source (id);
......
......@@ -9,6 +9,8 @@ ALTER TABLE track_mbid ADD CONSTRAINT track_mbid_pkey PRIMARY KEY (id);
ALTER TABLE track_mbid_source ADD CONSTRAINT track_mbid_source_pkey PRIMARY KEY (id);
ALTER TABLE track_puid ADD CONSTRAINT track_puid_pkey PRIMARY KEY (id);
ALTER TABLE track_puid_source ADD CONSTRAINT track_puid_source_pkey PRIMARY KEY (id);
ALTER TABLE track_meta ADD CONSTRAINT track_meta_pkey PRIMARY KEY (id);
ALTER TABLE track_meta_source ADD CONSTRAINT track_meta_source_pkey PRIMARY KEY (id);
ALTER TABLE submission ADD CONSTRAINT submission_pkey PRIMARY KEY (id);
ALTER TABLE stats ADD CONSTRAINT stats_pkey PRIMARY KEY (id);
ALTER TABLE stats_top_accounts ADD CONSTRAINT stats_top_accounts_pkey PRIMARY KEY (id);
......
......@@ -50,7 +50,6 @@ CREATE TABLE fingerprint (
format_id int,
created timestamp with time zone NOT NULL DEFAULT current_timestamp,
track_id int NOT NULL,
meta_id int,
hash_full bytea,
hash_query bytea,
submission_count int NOT NULL
......@@ -106,6 +105,22 @@ CREATE TABLE track_puid_source (
created timestamp with time zone DEFAULT current_timestamp
);
CREATE TABLE track_meta (
id serial NOT NULL,
track_id int NOT NULL,
meta_id int NOT NULL,
created timestamp with time zone DEFAULT current_timestamp,
submission_count int NOT NULL
);
CREATE TABLE track_meta_source (
id serial NOT NULL,
track_meta_id int NOT NULL,
submission_id int NOT NULL,
source_id int NOT NULL,
created timestamp with time zone DEFAULT current_timestamp
);
CREATE TABLE submission (
id serial NOT NULL,
fingerprint int[] NOT NULL,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment