Commit 2b4dd56e authored by Mark Heppner's avatar Mark Heppner

Refactor type handling

parent 2704c87c
......@@ -14,11 +14,7 @@
import logging
import collections
import base64
import datetime
from decimal import Decimal
# TODO what are the other types used for, other than tests?
from phoenixdb.types import Binary
from phoenixdb.types import TypeHelper
from phoenixdb.errors import OperationalError, NotSupportedError, ProgrammingError, InternalError
from phoenixdb.calcite import common_pb2
......@@ -31,35 +27,6 @@ ColumnDescription = collections.namedtuple('ColumnDescription', 'name type_code
"""Named tuple for representing results from :attr:`Cursor.description`."""
def time_from_java_sql_time(n):
dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
return dt.time()
def time_to_java_sql_time(t):
return ((t.hour * 60 + t.minute) * 60 + t.second) * 1000 + t.microsecond / 1000
def date_from_java_sql_date(n):
return datetime.date(1970, 1, 1) + datetime.timedelta(days=n)
def date_to_java_sql_date(d):
if isinstance(d, datetime.datetime):
d = d.date()
td = d - datetime.date(1970, 1, 1)
return td.days
def datetime_from_java_sql_timestamp(n):
return datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
def datetime_to_java_sql_timestamp(d):
td = d - datetime.datetime(1970, 1, 1)
return td.microseconds / 1000 + (td.seconds + td.days * 24 * 3600) * 1000
class Cursor(object):
"""Database cursor for executing queries and iterating over results.
......@@ -158,131 +125,21 @@ class Cursor(object):
self._connection._client.closeStatement(self._connection._id, self._id)
self._id = id
def _convert_class(self, class_name):
"""Converts a given Java class name.
:param class_name:
The JDBC class name.
:returns: tuple ``(rep, field_name, cast_type, mutate_type)``
WHERE
``rep`` is a ``common_pb2.Rep`` type
``field_name`` is the field name in ``common_pb2.TypedValue``
``cast_type`` is the method to cast the JDBC value to the Python value
``mutate_type`` is the method to mutate the Python value to the JDBC value
"""
rep = None
field_name = None
cast_type = None
mutate_type = None
# TODO move this to a separate class?
if class_name == 'java.lang.String':
rep = common_pb2.STRING
field_name = 'string_value'
cast_type = str
elif class_name == 'java.math.BigDecimal':
rep = common_pb2.BIG_DECIMAL
field_name = 'string_value'
cast_type = Decimal
elif class_name == 'java.lang.Boolean':
rep = common_pb2.BOOLEAN
field_name = 'boolean_value'
cast_type = bool
elif class_name == 'java.lang.Byte':
rep = common_pb2.BYTE
field_name = 'number_value'
cast_type = bytes
elif class_name == 'java.lang.Short':
rep = common_pb2.SHORT
field_name = 'number_value'
cast_type = int
elif class_name == 'java.lang.Integer':
rep = common_pb2.INTEGER
field_name = 'number_value'
cast_type = int
elif class_name == 'java.lang.Long':
rep = common_pb2.LONG
field_name = 'number_value'
cast_type = int
elif class_name == 'java.lang.Float':
rep = common_pb2.FLOAT
field_name = 'double_value'
cast_type = float
elif class_name == 'java.lang.Double':
rep = common_pb2.DOUBLE
field_name = 'double_value'
cast_type = float
elif class_name == '[B':
rep = common_pb2.BYTE_STRING
field_name = 'bytes_value'
cast_type = base64.b64decode
mutate_type = Binary
elif class_name == 'java.sql.Date':
rep = common_pb2.JAVA_SQL_DATE
field_name = 'number_value'
cast_type = date_from_java_sql_date
mutate_type = date_to_java_sql_date
elif class_name == 'java.sql.Time':
rep = common_pb2.JAVA_SQL_TIME
field_name = 'number_value'
cast_type = time_from_java_sql_time
mutate_type = time_to_java_sql_time
elif class_name == 'java.sql.Timestamp':
rep = common_pb2.JAVA_SQL_TIMESTAMP
field_name = 'number_value'
cast_type = datetime_from_java_sql_timestamp
mutate_type = datetime_to_java_sql_timestamp
# TODO should this be org.apache.phoenix.schema.types.PhoenixArray
# TODO arrays and structs are handled differently
#elif class_name == 'java.lang.Array':
# rep = common_pb2.ARRAY
#elif class_name == 'java.lang.Struct':
# rep = common_pb2.STRUCT
else:
rep = common_pb2.OBJECT
return rep, field_name, cast_type, mutate_type
def _set_signature(self, signature):
self._signature = signature
self._column_data_types = []
self._parameter_data_types = []
if signature is None:
return
identity = lambda value: value
for i, column in enumerate(signature.columns):
dtype = self._convert_class(column.column_class_name)
dtype = TypeHelper.from_rep(column.type.rep)
self._column_data_types.append(dtype)
for parameter in signature.parameters:
dtype = self._convert_class(parameter.class_name)
dtype = TypeHelper.from_phoenix(parameter.type_name)
self._parameter_data_types.append(dtype)
def _transform_parameters(self, parameters):
typed_parameters = []
for value, data_type in zip(parameters, self._parameter_data_types):
typed_value = common_pb2.TypedValue()
if value is None:
typed_value.null = True
typed_value.type = common_pb2.NULL
else:
typed_value.null = False
# use the mutator function
if data_type[3] is not None:
value = data_type[3](value)
# set the Rep field
typed_value.type = data_type[0]
# set the field_name to the value
setattr(typed_value, data_type[1], value)
typed_parameters.append(typed_value)
return typed_parameters
def _set_frame(self, frame):
self._frame = frame
self._pos = None
......@@ -309,6 +166,28 @@ class Cursor(object):
self._set_frame(result.first_frame)
self._updatecount = result.update_count
def _transform_parameters(self, parameters):
typed_parameters = []
for value, data_type in zip(parameters, self._parameter_data_types):
field_name, rep, mutate = data_type
typed_value = common_pb2.TypedValue()
if value is None:
typed_value.null = True
typed_value.type = common_pb2.NULL
else:
typed_value.null = False
# use the mutator function
if mutate is not None:
value = mutate(value)
typed_value.type = rep
setattr(typed_value, field_name, value)
typed_parameters.append(typed_value)
return typed_parameters
def execute(self, operation, parameters=None):
if self._closed:
raise ProgrammingError('the cursor is already closed')
......@@ -345,20 +224,6 @@ class Cursor(object):
self._transform_parameters(parameters),
maxRowCount=0)
def fetchone(self):
if self._frame is None:
raise ProgrammingError('no select statement was executed')
if self._pos is None:
return None
rows = self._frame.rows
row = self._transform_row(rows[self._pos])
self._pos += 1
if self._pos >= len(rows):
self._pos = None
if not self._frame.done:
self._fetch_next_frame()
return row
def _transform_row(self, row):
"""Transforms a Row into Python values.
......@@ -368,26 +233,42 @@ class Cursor(object):
:returns:
A list of values casted into the correct Python types.
"""
tmp = []
tmp_row = []
for i, column in enumerate(row.value):
# TODO handle arrays, structs
if column.has_array_value:
pass
elif column.scalar_value.null:
tmp.append(None)
tmp_row.append(None)
else:
dtype = self._column_data_types[i]
field_name, cast = self._column_data_types[i]
# get the value from the field_name
value = getattr(column.scalar_value, dtype[1])
# TODO handle None
value = getattr(column.scalar_value, field_name)
# cast the value
# TODO try/catch the casting
value = dtype[2](value)
if cast is not None:
value = cast(value)
tmp_row.append(value)
return tmp_row
tmp.append(value)
return tmp
def fetchone(self):
if self._frame is None:
raise ProgrammingError('no select statement was executed')
if self._pos is None:
return None
rows = self._frame.rows
row = self._transform_row(rows[self._pos])
self._pos += 1
if self._pos >= len(rows):
self._pos = None
if not self._frame.done:
self._fetch_next_frame()
return row
def fetchmany(self, size=None):
if size is None:
......
......@@ -15,10 +15,13 @@
import time
import datetime
import base64
from decimal import Decimal
from phoenixdb.calcite import common_pb2
__all__ = [
'Date', 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks',
'Binary', 'STRING', 'BINARY', 'NUMBER', 'DATETIME', 'ROWID', 'BOOLEAN',
'Binary', 'REP_TYPES', 'REP_TYPES_MAP', 'PHOENIX_PARAMETERS', 'PHOENIX_PARAMETERS_MAP',
'TypeHelper',
]
......@@ -63,37 +66,185 @@ class _BinaryString(str):
pass
class ColumnType(object):
def __init__(self, eq_types):
self.eq_types = tuple(eq_types)
self.eq_types_set = set(eq_types)
def __cmp__(self, other):
if other in self.eq_types_set:
return 0
if other < self.eq_types:
return 1
else:
return -1
STRING = ColumnType(['VARCHAR', 'CHAR'])
"""Type object that can be used to describe string-based columns."""
BINARY = ColumnType(['BINARY', 'VARBINARY'])
"""Type object that can be used to describe (long) binary columns."""
NUMBER = ColumnType(['INTEGER', 'UNSIGNED_INT', 'BIGINT', 'UNSIGNED_LONG', 'TINYINT', 'UNSIGNED_TINYINT', 'SMALLINT', 'UNSIGNED_SMALLINT', 'FLOAT', 'UNSIGNED_FLOAT', 'DOUBLE', 'UNSIGNED_DOUBLE', 'DECIMAL'])
"""Type object that can be used to describe numeric columns."""
DATETIME = ColumnType(['TIME', 'DATE', 'TIMESTAMP', 'UNSIGNED_TIME', 'UNSIGNED_DATE', 'UNSIGNED_TIMESTAMP'])
"""Type object that can be used to describe date/time columns."""
ROWID = ColumnType([])
"""Only implemented for DB API 2.0 compatibility, not used."""
BOOLEAN = ColumnType(['BOOLEAN'])
"""Type object that can be used to describe boolean columns. This is a phoenixdb-specific extension."""
# XXX ARRAY
def time_from_java_sql_time(n):
dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
return dt.time()
def time_to_java_sql_time(t):
return ((t.hour * 60 + t.minute) * 60 + t.second) * 1000 + t.microsecond / 1000
def date_from_java_sql_date(n):
return datetime.date(1970, 1, 1) + datetime.timedelta(days=n)
def date_to_java_sql_date(d):
if isinstance(d, datetime.datetime):
d = d.date()
td = d - datetime.date(1970, 1, 1)
return td.days
def datetime_from_java_sql_timestamp(n):
return datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
def datetime_to_java_sql_timestamp(d):
td = d - datetime.datetime(1970, 1, 1)
return td.microseconds / 1000 + (td.seconds + td.days * 24 * 3600) * 1000
REP_TYPES = {
'bool_value': [
common_pb2.PRIMITIVE_BOOLEAN,
common_pb2.BOOLEAN,
],
'string_value': [
common_pb2.PRIMITIVE_CHAR,
common_pb2.CHARACTER,
common_pb2.STRING,
common_pb2.BIG_DECIMAL,
# TODO DECIMAL type (java.math.BigDecimal) is set to a Rep.OBJECT...why?
common_pb2.OBJECT,
],
'number_value': [
common_pb2.PRIMITIVE_BYTE,
common_pb2.PRIMITIVE_INT,
common_pb2.PRIMITIVE_SHORT,
common_pb2.PRIMITIVE_LONG,
common_pb2.BYTE,
common_pb2.INTEGER,
common_pb2.SHORT,
common_pb2.LONG,
common_pb2.BIG_INTEGER,
common_pb2.JAVA_SQL_TIME,
common_pb2.JAVA_SQL_TIMESTAMP,
common_pb2.JAVA_SQL_DATE,
common_pb2.JAVA_UTIL_DATE,
common_pb2.NUMBER,
],
'bytes_value': [
common_pb2.BYTE_STRING,
],
'double_value': [
common_pb2.PRIMITIVE_FLOAT,
common_pb2.PRIMITIVE_DOUBLE,
common_pb2.FLOAT,
common_pb2.DOUBLE,
]
}
"""Groups of Rep types."""
REP_TYPES_MAP = dict( (v, k) for k in REP_TYPES for v in REP_TYPES[k] )
"""Flips the available types to allow for faster lookup by Rep."""
PHOENIX_PARAMETERS = {
'bool_value': [
('BOOLEAN', common_pb2.BOOLEAN),
],
'string_value': [
('CHAR', common_pb2.CHARACTER),
('VARCHAR', common_pb2.STRING),
('DECIMAL', common_pb2.BIG_DECIMAL),
],
'number_value': [
('INTEGER', common_pb2.INTEGER),
('UNSIGNED_INT', common_pb2.INTEGER),
('BIGINT', common_pb2.LONG),
('UNSIGNED_LONG', common_pb2.LONG),
('TINYINT', common_pb2.BYTE),
('UNSIGNED_TINYINT', common_pb2.BYTE),
('SMALLINT', common_pb2.SHORT),
('UNSIGNED_SMALLINT', common_pb2.SHORT),
('DATE', common_pb2.JAVA_SQL_DATE),
('UNSIGNED_DATE', common_pb2.JAVA_SQL_DATE),
('TIME', common_pb2.JAVA_SQL_TIME),
('UNSIGNED_TIME', common_pb2.JAVA_SQL_TIME),
('TIMESTAMP', common_pb2.JAVA_SQL_TIMESTAMP),
('UNSIGNED_TIMESTAMP', common_pb2.JAVA_SQL_TIMESTAMP),
],
'bytes_value': [
('BINARY', common_pb2.BYTE_STRING),
('VARBINARY', common_pb2.BYTE_STRING),
],
'double_value': [
('FLOAT', common_pb2.FLOAT),
('UNSIGNED_FLOAT', common_pb2.FLOAT),
('DOUBLE', common_pb2.DOUBLE),
('UNSIGNED_DOUBLE', common_pb2.DOUBLE),
]
}
"""Map Phoenix types to Reps."""
PHOENIX_PARAMETERS_MAP = dict( (v[0], (k, v[1])) for k in PHOENIX_PARAMETERS for v in PHOENIX_PARAMETERS[k] )
"""Flips the Phoenix types to allow for faster lookup by type."""
class TypeHelper(object):
@staticmethod
def from_rep(rep):
"""Finds a method to cast from a Python value into a Phoenix type.
:param rep:
The Rep enum from ``common_pb2.Rep``.
=
:returns: tuple ``(field_name, cast_type)``
WHERE
``field_name`` is the attribute in ``common_pb2.TypedValue``
``cast_type`` is the method to cast from the Phoenix value to the Python value
"""
field_name = None
cast_type = None
if rep in REP_TYPES_MAP:
field_name = REP_TYPES_MAP[rep]
if rep == common_pb2.BIG_DECIMAL:
cast_type = Decimal
elif rep == common_pb2.BYTE:
cast_type = bytes
elif rep == common_pb2.BYTE_STRING:
cast_type = base64.b64decode
elif rep == common_pb2.JAVA_SQL_DATE:
cast_type = date_from_java_sql_date
elif rep == common_pb2.JAVA_SQL_TIME:
cast_type = time_from_java_sql_time
elif rep == common_pb2.JAVA_SQL_TIMESTAMP:
cast_type = datetime_from_java_sql_timestamp
return field_name, cast_type
@staticmethod
def from_phoenix(type_name):
"""Converts a Phoenix parameter type into a Rep and a mutator method.
:param type_name:
The JDBC type name from ``common_pb2.AvaticaParameter``.
=
:returns: tuple ``(field_name, rep, mutate_type)``
WHERE
``field_name`` is the attribute in ``common_pb2.TypedValue``
``rep`` is a ``common_pb2.Rep`` type
``mutate_type`` is the method to mutate the Python value to the JDBC value
"""
field_name = None
rep = None
mutate_type = None
if type_name in PHOENIX_PARAMETERS_MAP:
field_name, rep = PHOENIX_PARAMETERS_MAP[type_name]
if type_name == 'DECIMAL':
mutate_type = str
elif type_name == 'BINARY' or type_name == 'VARBINARY':
mutate_type = Binary
elif type_name == 'DATE':
mutate_type = date_to_java_sql_date
elif type_name == 'TIME':
mutate_type = time_to_java_sql_time
elif type_name == 'TIMESTAMP':
mutate_type = datetime_to_java_sql_timestamp
return field_name, rep, mutate_type
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment