From 1e0be41db7cc7d6425ecaca7b39e29148a78a690 Mon Sep 17 00:00:00 2001 From: Nandan Atur Date: Fri, 29 May 2020 12:39:13 -0700 Subject: [PATCH 1/2] We are consuming asynpg in our project. We create binary build using pyinstaller. Earlier we were using sqlite3, but now we have switched to postgres with asynpg. When we build binary using pyinstaller, we hit issue described below. Lets take hello world sample: root@test:/scripts/test# cat hello.py from asyncpg.exceptions import UniqueViolationError as IntegrityError print("hello world") Building Binary: root@mdcapbin-engine-01:/scripts/test# python3.7 -m PyInstaller --paths /scripts/test --paths /usr/local/lib/python3.7/site-packages --name hello.bin --onefile --distpath ./hello --key=hello --windowed hello.py --hidden-import=databases.backends --hidden-import=databases.backends.postgres --exclude-module MySQLdb --exclude-module psycopg2 --hidden-import=asyncpg.exceptions --hidden-import=asyncpg.pgproto.pgproto --hidden-import=asyncpg.protocol.protocol --hidden-import=asyncpg 35 INFO: PyInstaller: 3.6 35 INFO: Python: 3.7.0 36 INFO: Platform: Linux-3.10.0-1062.9.1.el7.x86_64-x86_64-with-debian-9.5 36 INFO: wrote /scripts/test/hello.bin.spec 38 INFO: UPX is not available. 39 INFO: Extending PYTHONPATH with paths ['/scripts/test', '/scripts/test', '/usr/local/lib/python3.7/site-packages', '/scripts/test'] 39 INFO: Will encrypt Python bytecode with key: 00000000000hello 39 INFO: Adding dependencies on pyi_crypto.py module 40 INFO: checking Analysis 47 INFO: Building because /usr/local/lib/python3.7/site-packages/asyncpg/pgproto/pgproto.cpython-37m-x86_64-linux-gnu.so changed 47 INFO: Initializing module dependency graph... 48 INFO: Caching module graph hooks... 52 INFO: Analyzing base_library.zip ... 2923 INFO: Caching module dependency graph... 3002 INFO: running Analysis Analysis-00.toc 3014 INFO: Analyzing /scripts/test/hello.py 4270 INFO: Analyzing hidden import 'databases.backends' 5602 INFO: Processing pre-find module path hook distutils 5603 INFO: distutils: retargeting to non-venv dir '/usr/local/lib/python3.7' 5971 INFO: Analyzing hidden import 'databases.backends.postgres' 6223 INFO: Analyzing hidden import 'asyncpg.pgproto.pgproto' 6223 INFO: Analyzing hidden import 'Crypto.Cipher._AES' 6279 INFO: Processing module hooks... 6279 INFO: Loading module hook "hook-Crypto.py"... 6295 INFO: Loading module hook "hook-distutils.py"... 6297 INFO: Loading module hook "hook-encodings.py"... 6335 INFO: Loading module hook "hook-pkg_resources.py"... 6754 INFO: Processing pre-safe import module hook win32com 6756 INFO: Excluding import '__main__' 6757 INFO: Removing import of __main__ from module pkg_resources 6757 INFO: Loading module hook "hook-pydoc.py"... 6757 INFO: Loading module hook "hook-sqlalchemy.py"... 6882 INFO: Found 3 sqlalchemy hidden imports 6882 WARNING: Hidden import "pysqlite2" not found! 7473 WARNING: Hidden import "sqlalchemy.sql.functions.func" not found! 7483 INFO: Import to be excluded not found: 'sqlalchemy.testing' 7483 INFO: Loading module hook "hook-sqlite3.py"... 7517 INFO: Loading module hook "hook-sysconfig.py"... 7524 INFO: Loading module hook "hook-xml.py"... 7767 INFO: Looking for ctypes DLLs 7820 INFO: Analyzing run-time hooks ... 7825 INFO: Including run-time hook 'pyi_rth_multiprocessing.py' 7829 INFO: Including run-time hook 'pyi_rth_pkgres.py' 7838 INFO: Looking for dynamic libraries 8276 INFO: Looking for eggs 8276 INFO: Using Python library /usr/local/lib/libpython3.7m.so.1.0 8283 INFO: Warnings written to /scripts/test/build/hello.bin/warn-hello.bin.txt 8326 INFO: Graph cross-reference written to /scripts/test/build/hello.bin/xref-hello.bin.html 8340 INFO: checking PYZ 8355 INFO: Building because toc changed 8355 INFO: Building PYZ (ZlibArchive) /scripts/test/build/hello.bin/PYZ-00.pyz 9268 INFO: Building PYZ (ZlibArchive) /scripts/test/build/hello.bin/PYZ-00.pyz completed successfully. 9277 INFO: checking PKG 9278 INFO: Building because toc changed 9278 INFO: Building PKG (CArchive) PKG-00.pkg 18509 INFO: Building PKG (CArchive) PKG-00.pkg completed successfully. 18513 INFO: Bootloader /usr/local/lib/python3.7/site-packages/PyInstaller/bootloader/Linux-64bit/run 18513 INFO: checking EXE 18514 INFO: Building because toc changed 18514 INFO: Building EXE from EXE-00.toc 18516 INFO: Appending archive to ELF section in EXE ./hello/hello.bin 18549 INFO: Building EXE from EXE-00.toc completed successfully. Run binary and hit Runtime Exception: root@mdcapbin-engine-01:/scripts/test# ./hello/hello.bin Traceback (most recent call last): File "hello.py", line 1, in from asyncpg.exceptions import UniqueViolationError as IntegrityError File "/usr/local/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 623, in exec_module exec(bytecode, module.__dict__) File "site-packages/asyncpg/__init__.py", line 8, in File "/usr/local/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 623, in exec_module exec(bytecode, module.__dict__) File "site-packages/asyncpg/connection.py", line 19, in File "/usr/local/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 623, in exec_module exec(bytecode, module.__dict__) File "site-packages/asyncpg/connect_utils.py", line 27, in File "/usr/local/lib/python3.7/site-packages/PyInstaller/loader/pyimod03_importers.py", line 623, in exec_module exec(bytecode, module.__dict__) File "site-packages/asyncpg/protocol/__init__.py", line 8, in File "asyncpg/protocol/protocol.pyx", line 1, in init asyncpg.protocol.protocol File "asyncpg/pgproto/./buffer.pyx", line 12, in init asyncpg.pgproto.pgproto ImportError: cannot import name exceptions [1174] Failed to execute script hello The fix is as described in this change. --- buffer.pyx | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/buffer.pyx b/buffer.pyx index 6957e93..3fda704 100644 --- a/buffer.pyx +++ b/buffer.pyx @@ -9,8 +9,8 @@ from libc.string cimport memcpy import collections -from . import exceptions - +class BufferError(Exception): + pass @cython.no_gc_clear @cython.final @@ -31,7 +31,7 @@ cdef class WriteBuffer: self._size = 0 if self._view_count: - raise exceptions.BufferError( + raise BufferError( 'Deallocating buffer with attached memoryviews') def __getbuffer__(self, Py_buffer *buffer, int flags): @@ -47,7 +47,7 @@ cdef class WriteBuffer: cdef inline _check_readonly(self): if self._view_count: - raise exceptions.BufferError('the buffer is in read-only mode') + raise BufferError('the buffer is in read-only mode') cdef inline _ensure_alloced(self, ssize_t extra_length): cdef ssize_t new_size = extra_length + self._length @@ -90,7 +90,7 @@ cdef class WriteBuffer: cdef inline start_message(self, char type): if self._length != 0: - raise exceptions.BufferError( + raise BufferError( 'cannot start_message for a non-empty buffer') self._ensure_alloced(5) self._message_mode = 1 @@ -103,12 +103,12 @@ cdef class WriteBuffer: self._check_readonly() if not self._message_mode: - raise exceptions.BufferError( + raise BufferError( 'end_message can only be called with start_message') if self._length < 5: - raise exceptions.BufferError('end_message: buffer is too small') + raise BufferError('end_message: buffer is too small') if mlen > _MAXINT32: - raise exceptions.BufferError('end_message: message is too large') + raise BufferError('end_message: message is too large') hton.pack_int32(&self._buf[1], mlen) return self @@ -164,7 +164,7 @@ cdef class WriteBuffer: cpython.PyBytes_AsStringAndSize(data, &buf, &size) if size > _MAXINT32: - raise exceptions.BufferError('string is too large') + raise BufferError('string is too large') # `size` does not account for the NULL at the end. self.write_int32(size) self.write_cstr(buf, size) @@ -252,7 +252,7 @@ cdef class ReadBuffer: bytes data_bytes if not cpython.PyBytes_CheckExact(data): - raise exceptions.BufferError('feed_data: bytes object expected') + raise BufferError('feed_data: bytes object expected') # Uncomment the below code to test code paths that # read single int/str/bytes sequences are split over @@ -284,9 +284,9 @@ cdef class ReadBuffer: cdef inline _ensure_first_buf(self): if PG_DEBUG: if self._len0 == 0: - raise exceptions.BufferError('empty first buffer') + raise BufferError('empty first buffer') if self._length == 0: - raise exceptions.BufferError('empty buffer') + raise BufferError('empty buffer') if self._pos0 == self._len0: self._switch_to_next_buf() @@ -306,7 +306,7 @@ cdef class ReadBuffer: if PG_DEBUG: if self._len0 < 1: - raise exceptions.BufferError( + raise BufferError( 'debug: second buffer of ReadBuffer is empty') cdef inline const char* _try_read_bytes(self, ssize_t nbytes): @@ -394,13 +394,13 @@ cdef class ReadBuffer: return cpython.PyBytes_FromStringAndSize(cbuf, nbytes) if nbytes > self._length: - raise exceptions.BufferError( + raise BufferError( 'not enough data to read {} bytes'.format(nbytes)) if self._current_message_ready: self._current_message_len_unread -= nbytes if self._current_message_len_unread < 0: - raise exceptions.BufferError('buffer overread') + raise BufferError('buffer overread') result = cpython.PyBytes_FromStringAndSize(NULL, nbytes) buf = cpython.PyBytes_AS_STRING(result) @@ -410,7 +410,7 @@ cdef class ReadBuffer: cdef bytes read_len_prefixed_bytes(self): cdef int32_t size = self.read_int32() if size < 0: - raise exceptions.BufferError( + raise BufferError( 'negative length for a len-prefixed bytes value') if size == 0: return b'' @@ -423,7 +423,7 @@ cdef class ReadBuffer: size = self.read_int32() if size < 0: - raise exceptions.BufferError( + raise BufferError( 'negative length for a len-prefixed bytes value') if size == 0: @@ -453,13 +453,13 @@ cdef class ReadBuffer: if PG_DEBUG: if not self._buf0: - raise exceptions.BufferError( + raise BufferError( 'debug: first buffer of ReadBuffer is empty') self._ensure_first_buf() first_byte = self._try_read_bytes(1) if first_byte is NULL: - raise exceptions.BufferError('not enough data to read one byte') + raise BufferError('not enough data to read one byte') return first_byte[0] @@ -504,7 +504,7 @@ cdef class ReadBuffer: cdef inline read_null_str(self): if not self._current_message_ready: - raise exceptions.BufferError( + raise BufferError( 'read_null_str only works when the message guaranteed ' 'to be in the buffer') @@ -542,7 +542,7 @@ cdef class ReadBuffer: self._current_message_len_unread -= nread if self._current_message_len_unread < 0: - raise exceptions.BufferError( + raise BufferError( 'read_null_str: buffer overread') return result @@ -555,7 +555,7 @@ cdef class ReadBuffer: self._current_message_len_unread -= nread if self._current_message_len_unread < 0: - raise exceptions.BufferError( + raise BufferError( 'read_null_str: buffer overread') self._ensure_first_buf() @@ -573,7 +573,7 @@ cdef class ReadBuffer: self._ensure_first_buf() cbuf = self._try_read_bytes(1) if cbuf == NULL: - raise exceptions.BufferError( + raise BufferError( 'failed to read one byte on a non-empty buffer') self._current_message_type = cbuf[0] @@ -611,7 +611,7 @@ cdef class ReadBuffer: cdef int32_t put_message(self) except -1: if not self._current_message_ready: - raise exceptions.BufferError( + raise BufferError( 'cannot put message: no message taken') self._current_message_ready = False return 0 @@ -634,7 +634,7 @@ cdef class ReadBuffer: cdef discard_message(self): if not self._current_message_ready: - raise exceptions.BufferError('no message to discard') + raise BufferError('no message to discard') if self._current_message_len_unread > 0: self._read_and_discard(self._current_message_len_unread) self._current_message_len_unread = 0 @@ -642,7 +642,7 @@ cdef class ReadBuffer: cdef bytes consume_message(self): if not self._current_message_ready: - raise exceptions.BufferError('no message to consume') + raise BufferError('no message to consume') if self._current_message_len_unread > 0: mem = self.read_bytes(self._current_message_len_unread) else: @@ -653,14 +653,14 @@ cdef class ReadBuffer: cdef redirect_messages(self, WriteBuffer buf, char mtype, int stop_at=0): if not self._current_message_ready: - raise exceptions.BufferError( + raise BufferError( 'consume_full_messages called on a buffer without a ' 'complete first message') if mtype != self._current_message_type: - raise exceptions.BufferError( + raise BufferError( 'consume_full_messages called with a wrong mtype') if self._current_message_len_unread != self._current_message_len - 4: - raise exceptions.BufferError( + raise BufferError( 'consume_full_messages called on a partially read message') cdef: From 6f659bceb74858876d33805df2ae9e9f9f13f5fe Mon Sep 17 00:00:00 2001 From: Nandan Atur Date: Fri, 29 May 2020 12:54:00 -0700 Subject: [PATCH 2/2] Removing redundant file exceptions.py --- exceptions.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 exceptions.py diff --git a/exceptions.py b/exceptions.py deleted file mode 100644 index 8bbab65..0000000 --- a/exceptions.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (C) 2016-present the asyncpg authors and contributors -# -# -# This module is part of asyncpg and is released under -# the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0 - - -class BufferError(Exception): - pass