Commit 83119f17 authored by Lukáš Lalinský's avatar Lukáš Lalinský

Merged in mheppner/python-phoenixdb (pull request #1)

Protobuf serialization support
parents f1bd4d14 80b83f03
......@@ -2,3 +2,5 @@
/doc/_build/
*.pyc
*.egg-info/
.vagrant/
......@@ -39,32 +39,6 @@ used the same way you would use any other SQL database from Python, for example:
cursor.execute("SELECT * FROM users")
print cursor.fetchall()
Phoenix versions
----------------
Multiple Phoenix versions are supported, but because there is no way in the protocol
to determine the version at runtime, you need to provide the version of Avatica
used in the Phoenix version that you are using. Below is a table of the correct
versions for the official Phoenix releases.
=============== =============== ================================
Phoenix version Avatica version Connection URL
=============== =============== ================================
4.4 1.2 ``http://localhost:8765/?v=1.2``
4.5 1.3 ``http://localhost:8765/?v=1.3``
4.6 1.3 ``http://localhost:8765/?v=1.3``
4.7 1.6 ``http://localhost:8765/?v=1.6``
=============== =============== ================================
Phoenix 4.7 uses a serialization based on Protocol Buffers (proto3) by default.
This version of Protocol Buffers does not even have a stable release
and is not generally available on Linux distributions.
This library only supports the older JSON serialization. In order for the library
to work with Phoenix 4.7, you need to start the query server like this::
./bin/queryserver.py start -Dphoenix.queryserver.serialization=JSON
Setting up a development environment
------------------------------------
......@@ -77,6 +51,8 @@ necessary requirements::
pip install -r requirements.txt
python setup.py develop
To create or update protobuf classes, change the tag in `gen-protobuf.sh` and execute. For Calcite >= 1.6, the path changes from `avatica/src/main/protobuf` to `avatica/core/src/main/protobuf`.
If you need a Phoenix server for experimenting, you can get one running
quickly using Vagrant::
......
#!/usr/bin/env bash
export CALCITE_VER=calcite-1.8.0
export CALCITE_DIR=calcite
rm -rf phoenixdb/$CALCITE_DIR
rm -rf calcite-tmp
git init calcite-tmp
cd calcite-tmp
git remote add origin https://github.com/apache/calcite/
git config core.sparsecheckout true
echo "avatica/core/src/main/protobuf/*" >> .git/info/sparse-checkout
git pull --depth=1 origin $CALCITE_VER
cd ..
mkdir -p phoenixdb/$CALCITE_DIR
protoc --proto_path=calcite-tmp/avatica/core/src/main/protobuf/ --python_out=phoenixdb/$CALCITE_DIR calcite-tmp/avatica/core/src/main/protobuf/*.proto
rm -rf calcite-tmp
echo '' >> phoenixdb/$CALCITE_DIR/__init__.py
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
class Connection(object):
"""Database connection.
You should not construct this object manually, use :func:`~phoenixdb.connect` instead.
"""
......@@ -83,6 +83,7 @@ class Connection(object):
transactions. Only defined for DB API 2.0 compatibility.
You need to use :attr:`autocommit` mode.
"""
# TODO can support be added for this?
if self._closed:
raise ProgrammingError('the connection is already closed')
......@@ -100,12 +101,12 @@ class Connection(object):
def set_session(self, autocommit=None, readonly=None):
"""Sets one or more parameters in the current connection.
:param autocommit:
Switch the connection to autocommit mode. With the current
version, you need to always enable this, because
:meth:`commit` is not implemented.
:param readonly:
Switch the connection to read-only mode.
"""
......@@ -115,8 +116,9 @@ class Connection(object):
if readonly is not None:
props['readOnly'] = bool(readonly)
props = self._client.connectionSync(self._id, props)
self._autocommit = props['autoCommit']
self._readonly = props['readOnly']
self._autocommit = props.auto_commit
self._readonly = props.read_only
self._transactionisolation = props.transaction_isolation
@property
def autocommit(self):
......@@ -128,7 +130,7 @@ class Connection(object):
if self._closed:
raise ProgrammingError('the connection is already closed')
props = self._client.connectionSync(self._id, {'autoCommit': bool(value)})
self._autocommit = props['autoCommit']
self._autocommit = props.auto_commit
@property
def readonly(self):
......@@ -140,8 +142,18 @@ class Connection(object):
if self._closed:
raise ProgrammingError('the connection is already closed')
props = self._client.connectionSync(self._id, {'readOnly': bool(value)})
self._readonly = props['readOnly']
self._readonly = props.read_only
@property
def transactionisolation(self):
return self._transactionisolation
@readonly.setter
def transactionisolation(self, value):
if self._closed:
raise ProgrammingError('the connection is already closed')
props = self._client.connectionSync(self._id, {'transactionIsolation': bool(value)})
self._transactionisolation = props.transaction_isolation
for name in errors.__all__:
setattr(Connection, name, getattr(errors, name))
This diff is collapsed.
......@@ -3,7 +3,6 @@ import datetime
import phoenixdb
from decimal import Decimal
from phoenixdb.tests import DatabaseTestCase
from phoenixdb.avatica import AVATICA_1_2_0, AVATICA_1_3_0, AVATICA_1_4_0
class TypesTest(DatabaseTestCase):
......@@ -20,12 +19,12 @@ class TypesTest(DatabaseTestCase):
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
self.assertEqual(cursor.description[1].type_code, phoenixdb.NUMBER)
self.assertEqual(cursor.fetchall(), [[1, 1], [2, None], [3, 1], [4, None], [5, min_value], [6, max_value]])
if self.conn._client.version >= AVATICA_1_4_0:
self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, {})".format(min_value - 1))
self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, {})".format(max_value + 1))
# XXX The server silently truncates the values
#self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, ?)", [min_value - 1])
#self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, ?)", [max_value + 1])
self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, {})".format(min_value - 1))
self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, {})".format(max_value + 1))
# XXX The server silently truncates the values
#self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, ?)", [min_value - 1])
#self.assertRaises(self.conn.DatabaseError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, ?)", [max_value + 1])
def test_integer(self):
self.checkIntType("integer", -2147483648, 2147483647)
......@@ -84,8 +83,6 @@ class TypesTest(DatabaseTestCase):
self.checkFloatType("unsigned_double", 0, 1.7976931348623158E+308)
def test_decimal(self):
if self.conn._client.version < AVATICA_1_4_0:
raise unittest.SkipTest('decimal only works correctly with Calcite >= 1.4.0: https://issues.apache.org/jira/browse/CALCITE-795')
self.createTable("phoenixdb_test_tbl1", "id integer primary key, val decimal(8,3)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 33333.333)")
......@@ -172,7 +169,6 @@ class TypesTest(DatabaseTestCase):
[2, datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)],
])
@unittest.skip("broken")
def test_date_null(self):
self.createTable("phoenixdb_test_tbl1", "id integer primary key, val date")
with self.conn.cursor() as cursor:
......@@ -224,8 +220,6 @@ class TypesTest(DatabaseTestCase):
self.assertEqual(cursor.fetchall(), [[1, 'abc'], [2, None], [3, 'abc'], [4, None], [5, None], [6, None]])
def test_varchar_very_long(self):
if self.conn._client.version < AVATICA_1_4_0:
raise unittest.SkipTest('long requests only work with Calcite >= 1.4.0: https://issues.apache.org/jira/browse/CALCITE-780')
self.createTable("phoenixdb_test_tbl1", "id integer primary key, val varchar")
with self.conn.cursor() as cursor:
value = '1234567890' * 1000
......@@ -246,7 +240,6 @@ class TypesTest(DatabaseTestCase):
self.assertEqual(cursor.fetchall(), [[1, 'ab'], [2, None], [3, 'ab'], [4, None], [5, None], [6, None]])
self.assertRaises(self.conn.DataError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, 'abc')")
@unittest.skip("broken")
def test_char_null(self):
self.createTable("phoenixdb_test_tbl1", "id integer primary key, val char(2)")
with self.conn.cursor() as cursor:
......@@ -255,7 +248,7 @@ class TypesTest(DatabaseTestCase):
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (5, '')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (6, ?)", [''])
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
self.assertEqual(cursor.fetchall(), [[1, 'abc'], [2, None], [3, 'abc'], [4, None], [5, None], [6, None]])
self.assertEqual(cursor.fetchall(), [[2, None], [4, None], [5, None], [6, None]])
self.assertRaises(self.conn.DataError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, 'abc')")
def test_char(self):
......@@ -305,4 +298,3 @@ class TypesTest(DatabaseTestCase):
[1, [1,2]],
[2, [2,3]],
])
......@@ -14,11 +14,13 @@
import time
import datetime
import base64
from decimal import Decimal
from phoenixdb.calcite import common_pb2
__all__ = [
'Date', 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks',
'Binary', 'STRING', 'BINARY', 'NUMBER', 'DATETIME', 'ROWID', 'BOOLEAN',
'JAVA_CLASSES', 'JAVA_CLASSES_MAP', 'TypeHelper',
]
......@@ -54,13 +56,36 @@ def TimestampFromTicks(ticks):
def Binary(value):
"""Constructs an object capable of holding a binary (long) string value."""
if isinstance(value, _BinaryString):
return value
return _BinaryString(base64.b64encode(value))
return value
class _BinaryString(str):
pass
def time_from_java_sql_time(n):
dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
return dt.time()
def time_to_java_sql_time(t):
return ((t.hour * 60 + t.minute) * 60 + t.second) * 1000 + t.microsecond / 1000
def date_from_java_sql_date(n):
return datetime.date(1970, 1, 1) + datetime.timedelta(days=n)
def date_to_java_sql_date(d):
if isinstance(d, datetime.datetime):
d = d.date()
td = d - datetime.date(1970, 1, 1)
return td.days
def datetime_from_java_sql_timestamp(n):
return datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=n)
def datetime_to_java_sql_timestamp(d):
td = d - datetime.datetime(1970, 1, 1)
return td.microseconds / 1000 + (td.seconds + td.days * 24 * 3600) * 1000
class ColumnType(object):
......@@ -97,3 +122,68 @@ BOOLEAN = ColumnType(['BOOLEAN'])
"""Type object that can be used to describe boolean columns. This is a phoenixdb-specific extension."""
# XXX ARRAY
JAVA_CLASSES = {
'bool_value': [
('java.lang.Boolean', common_pb2.BOOLEAN, None, None),
],
'string_value': [
('java.lang.Character', common_pb2.CHARACTER, None, None),
('java.lang.String', common_pb2.STRING, None, None),
('java.math.BigDecimal', common_pb2.BIG_DECIMAL, str, Decimal),
],
'number_value': [
('java.lang.Integer', common_pb2.INTEGER, None, int),
('java.lang.Short', common_pb2.SHORT, None, int),
('java.lang.Long', common_pb2.LONG, None, long),
('java.lang.Byte', common_pb2.BYTE, None, int),
('java.sql.Time', common_pb2.JAVA_SQL_TIME, time_to_java_sql_time, time_from_java_sql_time),
('java.sql.Date', common_pb2.JAVA_SQL_DATE, date_to_java_sql_date, date_from_java_sql_date),
('java.sql.Timestamp', common_pb2.JAVA_SQL_TIMESTAMP, datetime_to_java_sql_timestamp, datetime_from_java_sql_timestamp),
],
'bytes_value': [
('[B', common_pb2.BYTE_STRING, Binary, None),
],
'double_value': [
# if common_pb2.FLOAT is used, incorrect values are sent
('java.lang.Float', common_pb2.DOUBLE, float, float),
('java.lang.Double', common_pb2.DOUBLE, float, float),
]
}
"""Groups of Java classes."""
JAVA_CLASSES_MAP = dict( (v[0], (k, v[1], v[2], v[3])) for k in JAVA_CLASSES for v in JAVA_CLASSES[k] )
"""Flips the available types to allow for faster lookup by Java class.
This mapping should be structured as:
{
'java.math.BigDecimal': ('string_value', common_pb2.BIG_DECIMAL, str, Decimal),),
...
'<java class>': (<field_name>, <Rep enum>, <mutate_to function>, <cast_from function>),
}
"""
class TypeHelper(object):
@staticmethod
def from_class(klass):
"""Retrieves a Rep and functions to cast to/from based on the Java class.
:param klass:
The string of the Java class for the column or parameter.
:returns: tuple ``(field_name, rep, mutate_to, cast_from)``
WHERE
``field_name`` is the attribute in ``common_pb2.TypedValue``
``rep`` is the common_pb2.Rep enum
``mutate_to`` is the function to cast values into Phoenix values, if any
``cast_from`` is the function to cast from the Phoenix value to the Python value, if any
:raises:
NotImplementedError
"""
if klass not in JAVA_CLASSES_MAP:
raise NotImplementedError('type {} is not supported'.format(klass))
return JAVA_CLASSES_MAP[klass]
......@@ -2,15 +2,18 @@
set -e
HBASE_VERSION=1.1.5
# if using Phoenix <= 4.7, file names are "phoenix-<version>-HBase-<version>"
#HBASE_VERSION=1.1.5
#PHOENIX_VERSION=4.7.0-HBase-1.1
PHOENIX_VERSION=4.6.0-HBase-1.1
#PHOENIX_VERSION=4.4.0-HBase-1.1
#APACHE_MIRROR=http://archive.apache.org/dist/
# if using Phoenix >= 4.8, file names have changed to "apache-phoenix-<version>-HBase-<version>"
HBASE_VERSION=1.2.3
PHOENIX_VERSION=4.8.0-HBase-1.2
export DEBIAN_FRONTEND=noninteractive
echo "> Removing chef and puppet"
......@@ -41,16 +44,25 @@ fi
if [ ! -d /opt/phoenix ]
then
echo "> Downloading Phoenix $PHOENIX_VERSION"
wget --no-verbose -P /tmp -c -N $APACHE_MIRROR/phoenix/phoenix-$PHOENIX_VERSION/bin/phoenix-$PHOENIX_VERSION-bin.tar.gz
# Phoenix <= 4.7
#wget --no-verbose -P /tmp -c -N $APACHE_MIRROR/phoenix/phoenix-$PHOENIX_VERSION/bin/phoenix-$PHOENIX_VERSION-bin.tar.gz
# Phoenix >= 4.8
wget --no-verbose -P /tmp -c -N $APACHE_MIRROR/phoenix/apache-phoenix-$PHOENIX_VERSION/bin/apache-phoenix-$PHOENIX_VERSION-bin.tar.gz
echo "> Extracting Phoenix"
sudo mkdir /opt/phoenix
sudo chown vagrant:vagrant -R /opt/phoenix
tar xf /tmp/phoenix-$PHOENIX_VERSION-bin.tar.gz --strip-components=1 -C /opt/phoenix
# Phoenix <= 4.7
#tar xf /tmp/phoenix-$PHOENIX_VERSION-bin.tar.gz --strip-components=1 -C /opt/phoenix
# Phoenix >= 4.8
tar xf /tmp/apache-phoenix-$PHOENIX_VERSION-bin.tar.gz --strip-components=1 -C /opt/phoenix
fi
echo "> Linking Phoenix server JAR file to HBase lib directory"
ln -svfT /opt/phoenix/phoenix-$PHOENIX_VERSION-server.jar /opt/hbase/lib/phoenix-$PHOENIX_VERSION-server.jar
# Phoenix <= 4.7
#ln -svfT /opt/phoenix/phoenix-$PHOENIX_VERSION-server.jar /opt/hbase/lib/phoenix-$PHOENIX_VERSION-server.jar
# Phoenix >= 4.8
ln -svfT /opt/phoenix/apache-phoenix-$PHOENIX_VERSION-server.jar /opt/hbase/lib/phoenix-$PHOENIX_VERSION-server.jar
echo "> Setting JAVA_HOME for HBase"
perl -pi -e 's{^\#?\s*export\s*JAVA_HOME\s*=.*$}{export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64}' /opt/hbase/conf/hbase-env.sh
......
......@@ -32,4 +32,7 @@ setup(
'release': ('setup.py', version),
},
},
install_requires=[
'protobuf>=3.0.0',
]
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment