diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100755 index 0000000..26fdfdd --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,55 @@ +# Contributing to FINOS qPython Project +Thanks for taking the time to contribute! + +# Contributor License Agreement (CLA) +A CLA is a document that specifies how a project is allowed to use your +contribution; they are commonly used in many open source projects. + +**_All_ contributions to _all_ projects hosted by [FINOS](https://www.finos.org/) +must be made with a +[Foundation CLA](https://finosfoundation.atlassian.net/wiki/spaces/FINOS/pages/75530375/Contribution+Compliance+Requirements#ContributionComplianceRequirements-ContributorLicenseAgreement) +in place, and there are [additional legal requirements](https://finosfoundation.atlassian.net/wiki/spaces/FINOS/pages/75530375/Contribution+Compliance+Requirements#ContributionComplianceRequirements-LicenseInformation) +that must also be met.** + +Commits and pull requests to FINOS repositories will only be accepted from those contributors with an active, executed Individual Contributor License Agreement (ICLA) with FINOS OR who are covered under an existing and active Corporate Contribution License Agreement (CCLA) executed with FINOS. Commits from individuals not covered under an ICLA or CCLA will be flagged and blocked by the EasyCLA tool. Please note that some CCLAs require individuals/employees to be explicitly named on the CCLA. + +As a result, PRs submitted to the kdb+ Project cannot be accepted until you have a CLA in place with the Foundation. + +Need an ICLA? Unsure if you are covered under an existing CCLA? Email [help@finos.org](mailto:help@finos.org?subject=CLA) + + +# Contributing Issues + +## Prerequisites + +* [ ] Have you [searched for duplicates](https://github.com/finos-data-tech/kdb/issues?utf8=%E2%9C%93&q=)? A simple search for exception error messages or a summary of the unexpected behaviour should suffice. +* [ ] Are you running the latest version? +* [ ] Are you sure this is a bug or missing capability? + +## Raising an Issue +* Create your issue [here](https://github.com/finos-data-tech/kdb/issues/new). +* New issues contain two templates in the description: bug report and enhancement request. Please pick the most appropriate for your issue, **then delete the other**. + * Please also tag the new issue with either "Bug" or "Enhancement". +* Please use [Markdown formatting](https://help.github.com/categories/writing-on-github/) +liberally to assist in readability. + * [Code fences](https://help.github.com/articles/creating-and-highlighting-code-blocks/) for exception stack traces and log entries, for example, massively improve readability. + +# Contributing Pull Requests (Code & Docs) +To make review of PRs easier, please: + + * Please make sure your PRs will merge cleanly - PRs that don't are unlikely to be accepted. + * For code contributions, follow the existing code layout. + * For documentation contributions, follow the general structure, language, and tone of the [existing docs](https://github.com/finos-data-tech/kdb/wiki). + * Keep commits small and cohesive - if you have multiple contributions, please submit them as independent commits (and ideally as independent PRs too). + * Reference issue #s if your PR has anything to do with an issue (even if it doesn't address it). + * Minimise non-functional changes (e.g. whitespace shenanigans). + * Ensure all new files include a header comment block containing the [Apache License v2.0 and your copyright information](http://www.apache.org/licenses/LICENSE-2.0#apply). + * If necessary (e.g. due to 3rd party dependency licensing requirements), update the [NOTICE file](https://github.com/finos-data-tech/kdb/blob/master/NOTICE) with any new attribution or other notices + + +## Commit and PR Messages + +* **Reference issues, wiki pages, and pull requests liberally!** +* Use the present tense ("Add feature" not "Added feature") +* Use the imperative mood ("Move button left..." not "Moves button left...") +* Limit the first line to 72 characters or less diff --git a/.github/workflows/cve-scanning.yml b/.github/workflows/cve-scanning.yml new file mode 100644 index 0000000..37d9a24 --- /dev/null +++ b/.github/workflows/cve-scanning.yml @@ -0,0 +1,36 @@ +name: Python CVE Scanning + +on: + + workflow_dispatch: + + pull_request: + paths: + - 'pyproject.toml' + - '.github/workflows/cve-scanning.yml' + push: + paths: + - 'pyproject.toml' + - '.github/workflows/cve-scanning.yml' + schedule: + # Run every day at 5am and 5pm + - cron: '0 5,17 * * *' + +jobs: + ci: + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + poetry-version: ["1.1.11"] + os: [ubuntu-18.04] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install safety + run: pip3 install safety + - name: Run safety check + run: safety check --full-report -r requirements.txt --policy-file safety-policy.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..686e047 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,32 @@ +name: QPython build/test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -U + - name: Build + run: | + python setup.py build_ext --inplace + - name: Test + run: | + env PYTHONPATH=. py.test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..5d9556a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,55 @@ +name: QPython publish + +on: + workflow_dispatch: + inputs: + version: + description: "Package version (x.y.z)" + required: true + prod: + description: "Use the real PyPI instead of the test one" + type: boolean + required: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Input check + env: + version: ${{ github.event.inputs.version }} + run: | + if ! [[ $version =~ ^[0-9.]+$ ]]; then echo invalid version; exit 1; fi + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade wheel + pip install -r requirements.txt -U + - name: Build + run: | + sed -i "s/@@VERSION_PLACEHOLDER@@/$version/g" qpython/__init__.py + python setup.py build_ext --inplace + python setup.py sdist + - name: Test + run: | + env PYTHONPATH=. py.test + - name: Build manylinux + uses: RalfG/python-wheels-manylinux-build@v0.3.4 + with: + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + build-requirements: 'cython numpy' + - name: Publish + env: + prodInput: ${{ github.event.inputs.prod }} + run: | + pip install --upgrade twine + find dist -name "*-linux*" -delete + ls -l dist + repoParam= + if [[ -z ${prodInput} ]]; then repoParam="--repository testpypi"; fi + twine upload ${repoParam} dist/* --username __token__ --password ${{ secrets.PYPI_API }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 900298b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python - -sudo: false - -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - -# command to install dependencies -install: pip install -r requirements.txt -U - -# command to run tests -script: - - python setup.py build_ext --inplace - - env PYTHONPATH=. py.test diff --git a/CHANGELOG.txt b/CHANGELOG.txt index fed2929..f73d24e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +------------------------------------------------------------------------------ + qPython3 1.0.0 [2021.06.18] +------------------------------------------------------------------------------ + + - Add support for kdb 3.5+ protocol (messages up to 2TB, 64-bit list sizes) + ------------------------------------------------------------------------------ qPython 2.0.0 [2019.01.01] ------------------------------------------------------------------------------ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cadc7bb..df40d19 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,12 @@ -To get started, sign the Contributor License Agreement. +## Contributing + +1. Fork it +2. Create your feature branch (`git checkout -b feature/fooBar`) +3. Read our [contribution guidelines](.github/CONTRIBUTING.md) and [Community Code of Conduct](https://www.finos.org/code-of-conduct) +4. Commit your changes (`git commit -am 'Add some fooBar'`) +5. Push to the branch (`git push origin feature/fooBar`) +6. Create a new Pull Request + +_NOTE:_ Commits and pull requests to FINOS repositories will only be accepted from those contributors with an active, executed Individual Contributor License Agreement (ICLA) with FINOS OR who are covered under an existing and active Corporate Contribution License Agreement (CCLA) executed with FINOS. Commits from individuals not covered under an ICLA or CCLA will be flagged and blocked by the EasyCLA tool. Please note that some CCLAs require individuals/employees to be explicitly named on the CCLA. + +*Need an ICLA? Unsure if you are covered under an existing CCLA? Email [help@finos.org](mailto:help@finos.org)* diff --git a/README.rst b/README.rst index 6cd51dd..4dd20b8 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,3 @@ -.. ATTENTION:: -This project is in maintenance mode. We may fix bugs, but no new features will be added in foreseeable future. - qPython ======= @@ -8,10 +5,10 @@ qPython is a Python library providing support for interprocess communication bet - Synchronous and asynchronous queries - Convenient asynchronous callbacks mechanism -- Support for kdb+ protocol and types: v3.0, v2.6, v<=2.5 +- Support for kdb+ protocol and types as of kdb+ v4.0 - Uncompression of the IPC data stream - Internal representation of data via numpy arrays (lists, complex types) and numpy data types (atoms) -- Supported on Python 2.7/3.4/3.5/3.6 and numpy 1.8+ +- Supported on Python 3.4/3.5/3.6 and numpy 1.8+ For more details please refer to the `documentation`_. @@ -21,9 +18,9 @@ Installation To install qPython from PyPI: -``$ pip install qpython`` +``$ pip install qpython3`` -**Please do not use old PyPI package name: exxeleron-qpython.** +**Please do not use old PyPI package name: qpython or exxeleron-qpython.** Building package @@ -68,6 +65,14 @@ Instructions: - Execute: ``py.test`` +CVE Scanning +~~~~~~~~~~~~ + +Configured GitHub Action as per the "Python" section of: + +- https://github.com/maoo/security-scanning + + Requirements ~~~~~~~~~~~~ diff --git a/pull_request_template.md b/pull_request_template.md new file mode 100644 index 0000000..f706d6c --- /dev/null +++ b/pull_request_template.md @@ -0,0 +1,3 @@ +THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER MATERIALS CONTRIBUTED IN CONNECTION WITH THIS SOFTWARE: +THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING THIS OR ANOTHER EQUIVALENT DISCLAIMER IN ADDITION TO ANY OTHER REQUIRED LICENSE TERMS. +ONLY THE SOFTWARE CODE AND OTHER MATERIALS CONTRIBUTED IN CONNECTION WITH THIS SOFTWARE, IF ANY, THAT ARE ATTACHED TO (OR OTHERWISE ACCOMPANY) THIS SUBMISSION (AND ORDINARY COURSE CONTRIBUTIONS OF FUTURES PATCHES THERETO) ARE TO BE CONSIDERED A CONTRIBUTION. NO OTHER SOFTWARE CODE OR MATERIALS ARE A CONTRIBUTION. diff --git a/qpython/__init__.py b/qpython/__init__.py index ff3f416..65d2678 100644 --- a/qpython/__init__.py +++ b/qpython/__init__.py @@ -17,7 +17,7 @@ __all__ = ['qconnection', 'qtype', 'qtemporal', 'qcollection'] -__version__ = '2.0.0' +__version__ = '@@VERSION_PLACEHOLDER@@' diff --git a/qpython/_pandas.py b/qpython/_pandas.py index 07e12a1..8cd3e0a 100644 --- a/qpython/_pandas.py +++ b/qpython/_pandas.py @@ -86,7 +86,7 @@ def _read_table(self, qtype = QTABLE): elif isinstance(data[i], bytes): # convert character list (represented as string) to numpy representation meta[column_name] = QSTRING - odict[column_name] = pandas.Series(list(data[i].decode()), dtype = numpy.str).replace(b' ', numpy.nan) + odict[column_name] = pandas.Series(list(data[i].decode()), dtype = str).replace(b' ', numpy.nan) elif isinstance(data[i], (list, tuple)): meta[column_name] = QGENERAL_LIST tarray = numpy.ndarray(shape = len(data[i]), dtype = numpy.dtype('O')) @@ -98,6 +98,7 @@ def _read_table(self, qtype = QTABLE): odict[column_name] = data[i] df = pandas.DataFrame(odict) + df._metadata = ["meta"] df.meta = meta return df else: @@ -111,7 +112,7 @@ def _read_list(self, qtype): qlist = QReader._read_list(self, qtype = qtype) if self._options.pandas: - if -abs(qtype) not in [QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]: + if -abs(qtype) not in [QBOOL, QMONTH, QDATE, QDATETIME, QMINUTE, QSECOND, QTIME, QTIMESTAMP, QTIMESPAN, QSYMBOL]: null = QNULLMAP[-abs(qtype)][1] ps = pandas.Series(data = qlist).replace(null, numpy.NaN) else: @@ -170,7 +171,7 @@ def _write_pandas_series(self, data, qtype = None): if qtype == QGENERAL_LIST: self._write_generic_list(data.values) elif qtype == QCHAR: - self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tostring()) + self._write_string(data.replace(numpy.nan, ' ').values.astype(numpy.string_).tobytes()) elif data.dtype.type not in (numpy.datetime64, numpy.timedelta64): data = data.fillna(QNULLMAP[-abs(qtype)][1]) data = data.values diff --git a/qpython/fastutils.pyx b/qpython/fastutils.pyx index 5e462e4..b614b24 100644 --- a/qpython/fastutils.pyx +++ b/qpython/fastutils.pyx @@ -17,9 +17,8 @@ import numpy cimport numpy -DTYPE = numpy.int -ctypedef numpy.int_t DTYPE_t -DTYPE8 = numpy.int +DTYPE = numpy.int64 +ctypedef numpy.int64_t DTYPE_t ctypedef numpy.uint8_t DTYPE8_t @@ -31,7 +30,6 @@ def uncompress(numpy.ndarray[DTYPE8_t] data, DTYPE_t uncompressed_size): cdef numpy.ndarray[DTYPE_t] ptrs = numpy.zeros(256, dtype = DTYPE) cdef numpy.ndarray[DTYPE8_t] uncompressed = numpy.zeros(uncompressed_size, dtype = numpy.uint8) - cdef numpy.ndarray[DTYPE_t] idx = numpy.arange(uncompressed_size, dtype = DTYPE) f = 0xff & data[0] @@ -41,7 +39,11 @@ def uncompress(numpy.ndarray[DTYPE8_t] data, DTYPE_t uncompressed_size): if f & i: r = ptrs[data[d]] n = 2 + data[d + 1] - uncompressed[idx[s:s + n]] = uncompressed[r:r + n] + if r+n<=s: + uncompressed[s:s + n] = uncompressed[r:r + n] + else: #overlapping slices! + for ii in range(n): + uncompressed[s+ii] = uncompressed[r+ii] ptrs[uncompressed[p] ^ uncompressed[pp]] = p if s == pp: diff --git a/qpython/qcollection.py b/qpython/qcollection.py index b9eb38e..1bb5870 100644 --- a/qpython/qcollection.py +++ b/qpython/qcollection.py @@ -16,7 +16,7 @@ from qpython.qtype import * # @UnusedWildImport from qpython import MetaData -from qpython.qtemporal import qtemporal, from_raw_qtemporal, to_raw_qtemporal +from qpython.qtemporal import qtemporal, QTemporal, from_raw_qtemporal, to_raw_qtemporal class QList(numpy.ndarray): @@ -27,7 +27,10 @@ def _meta_init(self, **meta): self.meta = MetaData(**meta) def __eq__(self, other): - return numpy.array_equal(self, other) + if (type(other) == QList or + type(other).__module__ == 'pandas.core.series' and type(other).__name__ == 'Series'): + return numpy.array_equal(self, other) + return super().__eq__(other) def __ne__(self, other): return not self.__eq__(other) @@ -66,6 +69,11 @@ def raw(self, idx): ''' return numpy.ndarray.__getitem__(self, idx) + def __repr__(self): + return 'QTemporalList({}, qtype={})'.format(numpy.array2string(self, separator=', ', formatter={"int": QTemporal.__str__}), -abs(self.meta.qtype)) + + def __str__(self): + return numpy.array2string(self, separator=', ', formatter={"int": QTemporal.__str__}) def get_list_qtype(array): @@ -397,7 +405,7 @@ def qtable(columns, data, **meta): class QKeyedTable(object): '''Represents a q keyed table. - :class:`.QKeyedTable` is built with two :class:`.QTable`\s, one representing + :class:`.QKeyedTable` is built with two :class:`.QTable`s, one representing keys and the other values. Keyed tables example: diff --git a/qpython/qconnection.py b/qpython/qconnection.py index c3054d8..ede969b 100644 --- a/qpython/qconnection.py +++ b/qpython/qconnection.py @@ -77,6 +77,7 @@ class QConnection(object): strings are encoded as q strings instead of chars, **Default**: ``False`` ''' + MAX_PROTOCOL_VERSION = 6 def __init__(self, host, port, username = None, password = None, timeout = None, encoding = 'latin-1', reader_class = None, writer_class = None, **options): self.host = host @@ -186,7 +187,7 @@ def _initialize(self): '''Performs a IPC protocol handshake.''' credentials = (self.username if self.username else '') + ':' + (self.password if self.password else '') credentials = credentials.encode(self._encoding) - self._connection.send(credentials + b'\3\0') + self._connection.send(credentials + bytes([self.MAX_PROTOCOL_VERSION, 0])) response = self._connection.recv(1) if len(response) != 1: @@ -199,7 +200,7 @@ def _initialize(self): self.close() raise QAuthenticationException('Connection denied.') - self._protocol_version = min(struct.unpack('B', response)[0], 3) + self._protocol_version = min(struct.unpack('B', response)[0], self.MAX_PROTOCOL_VERSION) def __str__(self): diff --git a/qpython/qreader.py b/qpython/qreader.py index 2dbdf54..32a7eb9 100644 --- a/qpython/qreader.py +++ b/qpython/qreader.py @@ -30,8 +30,6 @@ except: from qpython.utils import uncompress - - class QReaderException(Exception): ''' Indicates an error raised during data deserialization. @@ -42,15 +40,15 @@ class QReaderException(Exception): class QMessage(object): ''' - Represents a single message parsed from q protocol. + Represents a single message parsed from q protocol. Encapsulates data, message size, type, compression flag. - + :Parameters: - `data` - data payload - `message_type` (one of the constants defined in :class:`.MessageType`) - type of the message - - `message_size` (`integer`) - size of the message - - `is_compressed` (`boolean`) - indicates whether message is compressed + - `message_size` (`int`) - size of the message + - `compression_mode` (`int`) - indicates whether message is compressed, 1 for size<2GB, 2 for larger ''' @property @@ -70,9 +68,9 @@ def type(self): @property - def is_compressed(self): + def compression_mode(self): '''Indicates whether source message was compressed.''' - return self._is_compressed + return self._compression_mode @property @@ -81,29 +79,29 @@ def size(self): return self._size - def __init__(self, data, message_type, message_size, is_compressed): + def __init__(self, data, message_type, message_size, compression_mode): self._data = data self._type = message_type self._size = message_size - self._is_compressed = is_compressed + self._compression_mode = compression_mode def __str__(self, *args, **kwargs): - return 'QMessage: message type: %s, data size: %s, is_compressed: %s, data: %s' % (self._type, self._size, self._is_compressed, self._data) + return 'QMessage: message type: %s, data size: %s, compression_mode: %s, data: %s' % (self._type, self._size, self._compression_mode, self._data) class QReader(object): ''' Provides deserialization from q IPC protocol. - + :Parameters: - `stream` (`file object` or `None`) - data input stream - `encoding` (`string`) - encoding for characters parsing - + :Attrbutes: - - `_reader_map` - stores mapping between q types and functions - responsible for parsing into Python objects + - `_reader_map` - stores mapping between q types and functions + responsible for parsing into Python objects ''' _reader_map = {} @@ -119,24 +117,24 @@ def __init__(self, stream, encoding = 'latin-1'): def read(self, source = None, **options): ''' Reads and optionally parses a single message. - + :Parameters: - - `source` - optional data buffer to be read, if not specified data is + - `source` - optional data buffer to be read, if not specified data is read from the wrapped stream :Options: - - `raw` (`boolean`) - indicates whether read data should parsed or + - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are - backed by raw q representation (:class:`.QTemporalList`, - :class:`.QTemporal`) instances, otherwise are represented as + backed by raw q representation (:class:`.QTemporalList`, + :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - + :returns: :class:`.QMessage` - read data (parsed or raw byte form) along with meta information ''' message = self.read_header(source) - message.data = self.read_data(message.size, message.is_compressed, **options) + message.data = self.read_data(message.size, message.compression_mode, **options) return message @@ -144,14 +142,14 @@ def read(self, source = None, **options): def read_header(self, source = None): ''' Reads and parses message header. - + .. note:: :func:`.read_header` wraps data for further reading in internal - buffer - + buffer + :Parameters: - - `source` - optional data buffer to be read, if not specified data is + - `source` - optional data buffer to be read, if not specified data is read from the wrapped stream - + :returns: :class:`.QMessage` - read meta information ''' if self._stream: @@ -163,49 +161,50 @@ def read_header(self, source = None): self._buffer.endianness = '<' if self._buffer.get_byte() == 1 else '>' self._is_native = self._buffer.endianness == ('<' if sys.byteorder == 'little' else '>') message_type = self._buffer.get_byte() - message_compressed = self._buffer.get_byte() == 1 - # skip 1 byte - self._buffer.skip() + message_compression_mode = self._buffer.get_byte() + message_size_ext = self._buffer.get_byte() - message_size = self._buffer.get_int() - return QMessage(None, message_type, message_size, message_compressed) + message_size = self._buffer.get_uint() + message_size += message_size_ext << 32 + return QMessage(None, message_type, message_size, message_compression_mode) - def read_data(self, message_size, is_compressed = False, **options): + def read_data(self, message_size, compression_mode = 0, **options): ''' Reads and optionally parses data part of a message. - + .. note:: :func:`.read_header` is required to be called before executing the :func:`.read_data` - + :Parameters: - - `message_size` (`integer`) - size of the message to be read - - `is_compressed` (`boolean`) - indicates whether data is compressed + - `message_size` (`int`) - size of the message to be read + - `compression_mode` (`int`) - indicates whether data is compressed, 1 for <2GB, 2 for larger :Options: - - `raw` (`boolean`) - indicates whether read data should parsed or + - `raw` (`boolean`) - indicates whether read data should parsed or returned in raw byte form - `numpy_temporals` (`boolean`) - if ``False`` temporal vectors are - backed by raw q representation (:class:`.QTemporalList`, - :class:`.QTemporal`) instances, otherwise are represented as + backed by raw q representation (:class:`.QTemporalList`, + :class:`.QTemporal`) instances, otherwise are represented as `numpy datetime64`/`timedelta64` arrays and atoms, **Default**: ``False`` - + :returns: read data (parsed or raw byte form) ''' self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) - if is_compressed: + if compression_mode > 0: + comprHeaderLen = 4 if compression_mode == 1 else 8 if self._stream: - self._buffer.wrap(self._read_bytes(4)) - uncompressed_size = -8 + self._buffer.get_int() - compressed_data = self._read_bytes(message_size - 12) if self._stream else self._buffer.raw(message_size - 12) + self._buffer.wrap(self._read_bytes(comprHeaderLen)) + uncompressed_size = -8 + (self._buffer.get_uint() if compression_mode == 1 else self._buffer.get_long()) + compressed_data = self._read_bytes(message_size - (8+comprHeaderLen)) if self._stream else self._buffer.raw(message_size - (8+comprHeaderLen)) raw_data = numpy.frombuffer(compressed_data, dtype = numpy.uint8) if uncompressed_size <= 0: raise QReaderException('Error while data decompression.') - raw_data = uncompress(raw_data, numpy.intc(uncompressed_size)) - raw_data = numpy.ndarray.tostring(raw_data) + raw_data = uncompress(raw_data, numpy.int64(uncompressed_size)) + raw_data = numpy.ndarray.tobytes(raw_data) self._buffer.wrap(raw_data) elif self._stream: raw_data = self._read_bytes(message_size - 8) @@ -283,8 +282,9 @@ def _read_temporal(self, qtype): def _read_list(self, qtype): - self._buffer.skip() # ignore attributes - length = self._buffer.get_int() + attr = self._buffer.get_byte() + isLongLength = attr & 0x80 != 0 + length = self._buffer.get_long() if isLongLength else self._buffer.get_uint() conversion = PY_TYPE.get(-qtype, None) if qtype == QSYMBOL_LIST: @@ -414,7 +414,7 @@ def endianness(self): def endianness(self, endianness): ''' Sets the byte order (endianness) for reading from the buffer. - + :Parameters: - `endianness` (``<`` or ``>``) - byte order indicator ''' @@ -424,7 +424,7 @@ def endianness(self, endianness): def wrap(self, data): ''' Wraps the data in the buffer. - + :Parameters: - `data` - data to be wrapped ''' @@ -436,7 +436,7 @@ def wrap(self, data): def skip(self, offset = 1): ''' Skips reading of `offset` bytes. - + :Parameters: - `offset` (`integer`) - number of bytes to be skipped ''' @@ -451,10 +451,10 @@ def skip(self, offset = 1): def raw(self, offset): ''' Gets `offset` number of raw bytes. - + :Parameters: - `offset` (`integer`) - number of bytes to be retrieved - + :returns: raw bytes ''' new_position = self._position + offset @@ -470,11 +470,11 @@ def raw(self, offset): def get(self, fmt, offset = None): ''' Gets bytes from the buffer according to specified format or `offset`. - + :Parameters: - `fmt` (struct format) - conversion to be applied for reading - `offset` (`integer`) - number of bytes to be retrieved - + :returns: unpacked bytes ''' fmt = self.endianness + fmt @@ -485,7 +485,7 @@ def get(self, fmt, offset = None): def get_byte(self): ''' Gets a single byte from the buffer. - + :returns: single byte ''' return self.get('b') @@ -494,16 +494,34 @@ def get_byte(self): def get_int(self): ''' Gets a single 32-bit integer from the buffer. - + :returns: single integer ''' return self.get('i') + def get_uint(self): + ''' + Gets a single 32-bit unsigned integer from the buffer. + + :returns: single integer + ''' + return self.get('I') + + + def get_long(self): + ''' + Gets a single 64-bit integer from the buffer. + + :returns: single integer + ''' + return self.get('q') + + def get_symbol(self): ''' Gets a single, ``\\x00`` terminated string from the buffer. - + :returns: ``\\x00`` terminated string ''' new_position = self._data.find(b'\x00', self._position) @@ -519,10 +537,10 @@ def get_symbol(self): def get_symbols(self, count): ''' Gets ``count`` ``\\x00`` terminated strings from the buffer. - + :Parameters: - `count` (`integer`) - number of strings to be read - + :returns: list of ``\\x00`` terminated string read from the buffer ''' c = 0 diff --git a/qpython/qtemporal.py b/qpython/qtemporal.py index bb9042c..35deb42 100644 --- a/qpython/qtemporal.py +++ b/qpython/qtemporal.py @@ -67,6 +67,9 @@ def raw(self): return self._datetime def __str__(self): + return '%s' % (self._datetime) + + def __repr__(self): return '%s [%s]' % (self._datetime, self.meta) def __eq__(self, other): @@ -231,7 +234,7 @@ def _to_qmonth(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QMONTH).astype(int) if not dt == _NUMPY_NULL[QMONTH] else _QMONTH_NULL + return (dt - _EPOCH_QMONTH).astype(int) if not numpy.isnan(dt) else _QMONTH_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -250,7 +253,7 @@ def _to_qdate(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QDATE).astype(int) if not dt == _NUMPY_NULL[QDATE] else _QDATE_NULL + return (dt - _EPOCH_QDATE).astype(int) if not numpy.isnan(dt) else _QDATE_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -269,7 +272,7 @@ def _to_qdatetime(dt): if t_dt == numpy.float64: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not dt == _NUMPY_NULL[QDATETIME] else _QDATETIME_NULL + return (dt - _EPOCH_QDATETIME).astype(float) / _MILLIS_PER_DAY if not numpy.isnan(dt) else _QDATETIME_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -288,7 +291,7 @@ def _to_qminute(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QMINUTE] else _QMINUTE_NULL + return dt.astype(int) if not numpy.isnan(dt) else _QMINUTE_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -307,7 +310,7 @@ def _to_qsecond(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QSECOND] else _QSECOND_NULL + return dt.astype(int) if not numpy.isnan(dt) else _QSECOND_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -326,7 +329,7 @@ def _to_qtime(dt): if t_dt == numpy.int32: return dt elif t_dt == numpy.timedelta64: - return dt.astype(int) if not dt == _NUMPY_NULL[QTIME] else _QTIME_NULL + return dt.astype(int) if not numpy.isnan(dt) else _QTIME_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -345,7 +348,7 @@ def _to_qtimestamp(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.datetime64: - return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not dt == _NUMPY_NULL[QTIMESTAMP] else _QTIMESTAMP_NULL + return (dt - _EPOCH_TIMESTAMP).astype(longlong) if not numpy.isnan(dt) else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) @@ -364,7 +367,7 @@ def _to_qtimespan(dt): if t_dt == numpy.int64: return dt elif t_dt == numpy.timedelta64: - return dt.astype(longlong) if not dt == _NUMPY_NULL[QTIMESPAN] else _QTIMESTAMP_NULL + return dt.astype(longlong) if not numpy.isnan(dt) else _QTIMESTAMP_NULL else: raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt))) diff --git a/qpython/qtype.py b/qpython/qtype.py index ee44168..9a08ac0 100644 --- a/qpython/qtype.py +++ b/qpython/qtype.py @@ -187,7 +187,6 @@ # mapping of Python types to corresponding q atoms Q_TYPE = { bool : QBOOL, - numpy.bool : QBOOL, numpy.bool_ : QBOOL, numpy.byte : QBYTE, numpy.int16 : QSHORT, diff --git a/qpython/qwriter.py b/qpython/qwriter.py index d24fa5d..b402a64 100644 --- a/qpython/qwriter.py +++ b/qpython/qwriter.py @@ -34,7 +34,7 @@ class QWriterException(Exception): -ENDIANESS = '\1' if sys.byteorder == 'little' else '\0' +ENDIANNESS = '\1' if sys.byteorder == 'little' else '\0' class QWriter(object): @@ -81,7 +81,7 @@ def write(self, data, msg_type, **options): self._options = MetaData(**CONVERSION_OPTIONS.union_dict(**options)) # header and placeholder for message size - self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANESS, chr(msg_type))).encode(self._encoding)) + self._buffer.write(('%s%s\0\0\0\0\0\0' % (ENDIANNESS, chr(msg_type))).encode(self._encoding)) self._write(data) @@ -145,7 +145,10 @@ def _write_atom(self, data, qtype): try: self._buffer.write(struct.pack('b', qtype)) fmt = STRUCT_MAP[qtype] - self._buffer.write(struct.pack(fmt, data)) + if type(data) == numpy.bool_: + self._buffer.write(struct.pack(fmt, int(data))) + else: + self._buffer.write(struct.pack(fmt, data)) except KeyError: raise QWriterException('Unable to serialize type: %s' % data.__class__ if isinstance(data, object) else type(data)) @@ -258,7 +261,7 @@ def _write_list(self, data, qtype = None): if qtype == QGENERAL_LIST: self._write_generic_list(data) elif qtype == QCHAR: - self._write_string(data.tostring()) + self._write_string(data.tobytes()) else: self._buffer.write(struct.pack('=bxi', -qtype, len(data))) if data.dtype.type in (numpy.datetime64, numpy.timedelta64): @@ -277,5 +280,5 @@ def _write_list(self, data, qtype = None): for guid in data: self._buffer.write(guid.bytes) else: - self._buffer.write(data.tostring()) + self._buffer.write(data.tobytes()) diff --git a/qpython/utils.py b/qpython/utils.py index daebee3..e2b4aae 100644 --- a/qpython/utils.py +++ b/qpython/utils.py @@ -18,19 +18,18 @@ def uncompress(data, uncompressed_size): - _0 = numpy.intc(0) - _1 = numpy.intc(1) - _2 = numpy.intc(2) - _128 = numpy.intc(128) - _255 = numpy.intc(255) + _0 = numpy.int64(0) + _1 = numpy.int64(1) + _2 = numpy.int64(2) + _128 = numpy.int64(128) + _255 = numpy.int64(255) n, r, s, p = _0, _0, _0, _0 i, d = _1, _1 f = _255 & data[_0] - ptrs = numpy.zeros(256, dtype = numpy.intc) + ptrs = numpy.zeros(256, dtype = numpy.int64) uncompressed = numpy.zeros(uncompressed_size, dtype = numpy.uint8) - idx = numpy.arange(uncompressed_size, dtype = numpy.intc) while s < uncompressed_size: pp = p + _1 @@ -38,7 +37,11 @@ def uncompress(data, uncompressed_size): if f & i: r = ptrs[data[d]] n = _2 + data[d + _1] - uncompressed[idx[s:s + n]] = uncompressed[r:r + n] + if r+n<=s: + uncompressed[s:s + n] = uncompressed[r:r + n] + else: #overlapping slices! + for ii in range(n): + uncompressed[s+ii] = uncompressed[r+ii] ptrs[(uncompressed[p]) ^ (uncompressed[pp])] = p if s == pp: diff --git a/safety-policy.yml b/safety-policy.yml new file mode 100644 index 0000000..75a071d --- /dev/null +++ b/safety-policy.yml @@ -0,0 +1,8 @@ +security: + ignore-cvss-severity-below: 6 + ignore-cvss-unknown-severity: False + ignore-vulnerabilities: + 25853: # Example vulnerability ID + reason: Testing CVE suppression + expires: '2023-12-31' + continue-on-vulnerability-error: False diff --git a/setup.py b/setup.py index 08cf6b8..d5d1931 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ try: from Cython.Build import cythonize except ImportError: + print("unable to load cython, not using it") use_cython = False else: use_cython = True @@ -43,14 +44,14 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() -setup(name = 'qPython', +setup(name = 'qPython3', version = __version__, description = 'kdb+ interfacing library for Python', long_description=read('README.rst'), - author = 'exxeleron', - author_email = 'kdbtools@devnet.de', - url = 'http://github.com/exxeleron/qPython', + author = 'finos', + author_email = 'data-tech-kdb@finos.org', + url = 'https://github.com/finos/qPython', license = 'Apache License Version 2.0', ext_modules = ext_modules, @@ -71,8 +72,6 @@ def read(fname): 'Operating System :: POSIX', 'Operating System :: Unix', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', diff --git a/tests/pandas_test.py b/tests/pandas_test.py index b39d533..baacd92 100644 --- a/tests/pandas_test.py +++ b/tests/pandas_test.py @@ -50,7 +50,7 @@ (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', [b'quick', numpy.nan, b'fox', b'jumps', b'over', b'a lazy', b'dog']), - (b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = numpy.bool)), + (b'(0b;1b;0b)', {'data': pandas.Series(numpy.array([False, True, False], dtype = bool)), 'meta': MetaData(qtype = QBOOL_LIST) }), (b'(0x01;0x02;0xff)', {'data': pandas.Series(numpy.array([1, 2, 0xff], dtype = numpy.int8)), 'meta': MetaData(qtype = QBYTE_LIST) }), @@ -83,7 +83,7 @@ 'meta': MetaData(qtype = QMONTH_LIST) }), (b'2001.01.01 2000.05.01 0Nd', {'data': pandas.Series(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')), 'meta': MetaData(qtype = QDATE_LIST) }), - (b'2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')])), + (b'2000.01.04T05:36:57.600 0Nz', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')])), 'meta': MetaData(qtype = QDATETIME_LIST) }), (b'12:01 0Nu', {'data': pandas.Series(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')])), 'meta': MetaData(qtype = QMINUTE_LIST) }), @@ -91,7 +91,7 @@ 'meta': MetaData(qtype = QSECOND_LIST) }), (b'12:04:59.123 0Nt', {'data': pandas.Series(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')])), 'meta': MetaData(qtype = QTIME_LIST) }), - (b'2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')])), + (b'2000.01.04D05:36:57.600 0Np', {'data': pandas.Series(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')])), 'meta': MetaData(qtype = QTIMESTAMP_LIST) }), (b'0D05:36:57.600 0Nn', {'data': pandas.Series(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')])), 'meta': MetaData(qtype = QTIMESPAN_LIST) }), @@ -117,33 +117,33 @@ (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', {'data': pandas.DataFrame(OrderedDict((('name', pandas.Series(['Dent', 'Beeblebrox', 'Prefect'], dtype = numpy.string_)), ('iq', pandas.Series(numpy.array([98, 42, 126], dtype = numpy.int64))), - ('grade', pandas.Series(['a', ' ', 'c'], dtype = numpy.str).replace(b' ', numpy.nan)), + ('grade', pandas.Series(['a', ' ', 'c'], dtype = str).replace(b' ', numpy.nan)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'name': QSYMBOL_LIST, 'iq': QLONG_LIST, 'grade': QSTRING}) }), (b'1#([] sym:`x`x`x;str:" a")', {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)), - ('str', pandas.Series([' '], dtype = numpy.str).replace(b' ', numpy.nan)), + ('str', pandas.Series([' '], dtype = str).replace(b' ', numpy.nan)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}), 'single_char_strings': True}), (b'-1#([] sym:`x`x`x;str:" a")', {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x'], dtype = numpy.string_)), - ('str', pandas.Series(['a'], dtype = numpy.str)), + ('str', pandas.Series(['a'], dtype = str)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING}), 'single_char_strings': True}), (b'2#([] sym:`x`x`x`x;str:" aa")', {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)), - ('str', pandas.Series([' ', ' '], dtype = numpy.str).replace(b' ', numpy.nan)), + ('str', pandas.Series([' ', ' '], dtype = str).replace(b' ', numpy.nan)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}), (b'-2#([] sym:`x`x`x`x;str:" aa")', {'data': pandas.DataFrame(OrderedDict((('sym', pandas.Series(['x', 'x'], dtype = numpy.string_)), - ('str', pandas.Series(['a', 'a'], dtype = numpy.str).replace(b' ', numpy.nan)), + ('str', pandas.Series(['a', 'a'], dtype = str).replace(b' ', numpy.nan)), )) ), 'meta': MetaData(**{'qtype': QTABLE, 'sym': QSYMBOL_LIST, 'str': QSTRING})}), @@ -282,6 +282,7 @@ def test_reading_pandas(): if 'index' in value: meta = result.meta result = result.reset_index() + result._metadata = ["meta"] result.meta = meta if not 'compare_meta' in value or value['compare_meta']: @@ -309,6 +310,7 @@ def test_writing_pandas(): data = data.set_index(value['index']) if 'single_char_strings' in value: single_char_strings = value['single_char_strings'] + data._metadata = ["meta"] data.meta = value['meta'] else: data = value diff --git a/tests/qreader_test.py b/tests/qreader_test.py index c44363d..bf1027e 100644 --- a/tests/qreader_test.py +++ b/tests/qreader_test.py @@ -28,213 +28,11 @@ from collections import OrderedDict from qpython import qreader from qpython.qtype import * # @UnusedWildImport -from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QKeyedTable +from qpython.qcollection import qlist, QList, QTemporalList, QDictionary, qtable, QTable, QKeyedTable from qpython.qtemporal import qtemporal, QTemporal -EXPRESSIONS = OrderedDict(( - (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', - qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)), - (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), - (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), - (b'(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)), - (b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)), - (b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)), - (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)), - (b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)), - (b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)), - (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)), - (b'2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME)), - (b'0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)), - (b'12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)), - (b'12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)), - (b'0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)), - (b'12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)), - (b'12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)), - (b'0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)), - (b'12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)), - (b'12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)), - (b'0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)), - (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)), - (b'2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP)), - (b'0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)), - (b'0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)), - (b'0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)), - (b'0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)), - - (b'::', None), - (b'1+`', QException(b'type')), - (b'1', numpy.int64(1)), - (b'1i', numpy.int32(1)), - (b'-234h', numpy.int16(-234)), - (b'0b', numpy.bool_(False)), - (b'1b', numpy.bool_(True)), - (b'0x2a', numpy.byte(0x2a)), - (b'89421099511627575j', numpy.int64(long(89421099511627575))), - (b'5.5e', numpy.float32(5.5)), - (b'3.234', numpy.float64(3.234)), - (b'"0"', b'0'), - (b'"abc"', b'abc'), - (b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog'), - (b'`abc', numpy.string_('abc')), - (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), - (b'0Nh', qnull(QSHORT)), - (b'0N', qnull(QLONG)), - (b'0Ni', qnull(QINT)), - (b'0Nj', qnull(QLONG)), - (b'0Ne', qnull(QFLOAT)), - (b'0n', qnull(QDOUBLE)), - (b'" "', qnull(QSTRING)), - (b'`', qnull(QSYMBOL)), - (b'0Ng', qnull(QGUID)), - (b'()', []), - (b'(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)), - (b'(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)), - (b'(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - (b'1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)), - (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), - (b'(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)), - (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), - (b'3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), b'0bc', numpy.float32(5.5)]), - (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), - (b'`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), - (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', - [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']), - (b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', - [b'quick', b' ', b'fox', b'jumps', b'over', b'a lazy', b'dog']), - (b'{x+y}', QLambda('{x+y}')), - (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), - (b'insert [1]', QProjection([QFunction(0), numpy.int64(1)])), - (b'xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')), - (b'not', QFunction(0)), - (b'and', QFunction(0)), - (b'md5', QProjection([QFunction(0), numpy.int64(-15)])), - (b'any', QFunction(0)), - (b'save', QFunction(0)), - (b'raze', QFunction(0)), - (b'sums', QFunction(0)), - (b'prev', QFunction(0)), - - (b'(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), - qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST))), - (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), - qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))), - (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), - qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))), - (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], - qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))), - (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'], - [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), - (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), - [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [b'a', numpy.int64(2)]), numpy.float32(5.5)])), - - (b'flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)])), - (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', - qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)])), - (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', - qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - b"a c"])), - (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', - qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - [b"Arthur Dent", b"Zaphod Beeblebrox", b"Ford Prefect"]])), - (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', - qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - [b"Arthur Dent", b" ", b"Ford Prefect"]])), - (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), - qlist(numpy.array([3, 4]), qtype = QLONG_LIST), - qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]])), - (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), - qlist(numpy.array([3, 4]), qtype = QLONG_LIST), - qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])), - (b'1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b" "])), - (b'-1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b"a"])), - (b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b" "])), - (b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b"aa"])), - (b'([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), - qlist(numpy.array([]), qtype = QINT_LIST)])), - (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])), - (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), - qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) - ), - - )) - - -NUMPY_TEMPORAL_EXPRESSIONS = OrderedDict(( - (b'(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)), - (b'2001.01m', numpy.datetime64('2001-01', 'M')), - (b'0Nm', numpy.datetime64('NaT', 'M')), - (b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)), - (b'2001.01.01', numpy.datetime64('2001-01-01', 'D')), - (b'0Nd', numpy.datetime64('NaT', 'D')), - (b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)), - (b'2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms')), - (b'0Nz', numpy.datetime64('NaT', 'ms')), - (b'12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)), - (b'12:01', numpy.timedelta64(721, 'm')), - (b'0Nu', numpy.timedelta64('NaT', 'm')), - (b'12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)), - (b'12:05:00', numpy.timedelta64(43500, 's')), - (b'0Nv', numpy.timedelta64('nat', 's')), - (b'12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)), - (b'12:04:59.123', numpy.timedelta64(43499123, 'ms')), - (b'0Nt', numpy.timedelta64('NaT', 'ms')), - (b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)), - (b'2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns')), - (b'0Np', numpy.datetime64('NaT', 'ns')), - (b'0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)), - (b'0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')), - (b'0Nn', numpy.timedelta64('NaT', 'ns')), - (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - qtable(['pos', 'dates'], - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])), - )) - - - COMPRESSED_EXPRESSIONS = OrderedDict(( (b'1000#`q', qlist(numpy.array(['q'] * 1000), qtype=QSYMBOL_LIST)), (b'([] q:1000#`q)', qtable(qlist(numpy.array(['q']), qtype = QSYMBOL_LIST), @@ -276,18 +74,17 @@ def arrays_equal(left, right): def compare(left, right): - if type(left) in [float, numpy.float32, numpy.float64] and numpy.isnan(left): + if type(left) in [float, numpy.float32, numpy.float64, numpy.datetime64, numpy.timedelta64] and numpy.isnan(left): return numpy.isnan(right) - if type(left) == QTemporal and isinstance(left.raw, float) and numpy.isnan(left.raw): + elif type(left) == QTemporal and (isinstance(left.raw, float) or isinstance(left.raw, numpy.datetime64) or isinstance(left.raw, numpy.timedelta64)) and numpy.isnan(left.raw): return numpy.isnan(right.raw) - elif type(left) in [list, tuple, numpy.ndarray, QList, QTemporalList]: + elif type(left) in [list, tuple, numpy.ndarray, numpy.record, QList, QTemporalList, QTable]: return arrays_equal(left, right) elif type(left) == QFunction: return type(right) == QFunction else: return left == right - def test_reading(): BINARY = OrderedDict() @@ -303,7 +100,8 @@ def test_reading(): buffer_reader = qreader.QReader(None) print('Deserialization') - for query, value in iter(EXPRESSIONS.items()): + + def test_reading_one(query, value): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) @@ -315,7 +113,7 @@ def test_reading(): sys.stdout.write( ' %-75s' % query ) try: header = buffer_reader.read_header(source = buffer_.getvalue()) - result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed, raw = True) + result = buffer_reader.read_data(message_size = header.size, compression_mode = header.compression_mode, raw = True) assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) stream_reader = qreader.QReader(buffer_) @@ -323,10 +121,13 @@ def test_reading(): assert compare(buffer_.getvalue()[8:], result), 'raw reading failed: %s' % (query) result = buffer_reader.read(source = buffer_.getvalue()).data - assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) + if type(value) == QTemporalList: + assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, [x.raw for x in value], [x.raw for x in result]) + else: + assert compare(value, result), 'deserialization failed: %s, expected: %s actual: %s' % (query, value, result) header = buffer_reader.read_header(source = buffer_.getvalue()) - result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed) + result = buffer_reader.read_data(message_size = header.size, compression_mode = header.compression_mode) assert compare(value, result), 'deserialization failed: %s' % (query) buffer_.seek(0) @@ -339,7 +140,170 @@ def test_reading(): assert e.args == value.args print('.') - + test_reading_one(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', + qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)) + test_reading_one(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')) + test_reading_one(b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')) + test_reading_one(b'(2001.01m; 0Nm)', qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST)) + test_reading_one(b'2001.01m', qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH)) + test_reading_one(b'0Nm', qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH)) + test_reading_one(b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)) + test_reading_one(b'2001.01.01', qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE)) + test_reading_one(b'0Nd', qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE)) + test_reading_one(b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST)) + test_reading_one(b'2000.01.04T05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME)) + test_reading_one(b'0Nz', qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME)) + test_reading_one(b'12:01 0Nu', qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST)) + test_reading_one(b'12:01', qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE)) + test_reading_one(b'0Nu', qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE)) + test_reading_one(b'12:05:00 0Nv', qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST)) + test_reading_one(b'12:05:00', qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND)) + test_reading_one(b'0Nv', qtemporal(numpy.timedelta64('NaT', 's'), qtype=QSECOND)) + test_reading_one(b'12:04:59.123 0Nt', qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST)) + test_reading_one(b'12:04:59.123', qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME)) + test_reading_one(b'0Nt', qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME)) + test_reading_one(b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST)) + test_reading_one(b'2000.01.04D05:36:57.600', qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP)) + test_reading_one(b'0Np', qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP)) + test_reading_one(b'0D05:36:57.600 0Nn', qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST)) + test_reading_one(b'0D05:36:57.600', qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN)) + test_reading_one(b'0Nn', qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN)) + + test_reading_one(b'::', None) + test_reading_one(b'1+`', QException(b'type')) + test_reading_one(b'1', numpy.int64(1)) + test_reading_one(b'1i', numpy.int32(1)) + test_reading_one(b'-234h', numpy.int16(-234)) + test_reading_one(b'0b', numpy.bool_(False)) + test_reading_one(b'1b', numpy.bool_(True)) + test_reading_one(b'0x2a', numpy.byte(0x2a)) + test_reading_one(b'89421099511627575j', numpy.int64(long(89421099511627575))) + test_reading_one(b'5.5e', numpy.float32(5.5)) + test_reading_one(b'3.234', numpy.float64(3.234)) + test_reading_one(b'"0"', b'0') + test_reading_one(b'"abc"', b'abc') + test_reading_one(b'"quick brown fox jumps over a lazy dog"', b'quick brown fox jumps over a lazy dog') + test_reading_one(b'`abc', numpy.string_('abc')) + test_reading_one(b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')) + test_reading_one(b'0Nh', qnull(QSHORT)) + test_reading_one(b'0N', qnull(QLONG)) + test_reading_one(b'0Ni', qnull(QINT)) + test_reading_one(b'0Nj', qnull(QLONG)) + test_reading_one(b'0Ne', qnull(QFLOAT)) + test_reading_one(b'0n', qnull(QDOUBLE)) + test_reading_one(b'" "', qnull(QSTRING)) + test_reading_one(b'`', qnull(QSYMBOL)) + test_reading_one(b'0Ng', qnull(QGUID)) + test_reading_one(b'()', []), + test_reading_one(b'(0b;1b;0b)', qlist(numpy.array([False, True, False], dtype=numpy.bool_), qtype=QBOOL_LIST)) + test_reading_one(b'(0x01;0x02;0xff)', qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype=QBYTE_LIST)) + test_reading_one(b'(1h;2h;3h)', qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype=QSHORT_LIST)) + test_reading_one(b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)) + test_reading_one(b'1 2 3', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_reading_one(b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_reading_one(b'(1i;2i;3i)', qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype=QINT_LIST)) + test_reading_one(b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)) + test_reading_one(b'(1j;2j;3j)', qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_reading_one(b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_reading_one(b'(5.5e; 8.5e)', qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype=QFLOAT_LIST)) + test_reading_one(b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)) + test_reading_one(b'3.23 6.46', qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype=QDOUBLE_LIST)) + test_reading_one(b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)) + test_reading_one(b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), b'0bc', numpy.float32(5.5)]) + test_reading_one(b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]) + test_reading_one(b'`the`quick`brown`fox', qlist(numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=object), qtype=QSYMBOL_LIST)) + test_reading_one(b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=object), qtype=QSYMBOL_LIST)) + test_reading_one(b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=object), qtype=QSYMBOL_LIST)) + test_reading_one(b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', b'brown', b'fox', b'jumps', b'over', b'a lazy', b'dog']) + test_reading_one(b'("quick"; " "; "fox"; "jumps"; "over"; "a lazy"; "dog")', + [b'quick', b' ', b'fox', b'jumps', b'over', b'a lazy', b'dog']) + test_reading_one(b'{x+y}', QLambda('{x+y}')), + test_reading_one(b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])) + test_reading_one(b'insert [1]', QProjection([QFunction(0), numpy.int64(1)])) + test_reading_one(b'xbar', QLambda('k){x*y div x:$[16h=abs[@x];"j"$x;x]}')) + test_reading_one(b'not', QFunction(0)) + test_reading_one(b'and', QFunction(0)) + test_reading_one(b'md5', QProjection([QFunction(0), numpy.int64(-15)])) + test_reading_one(b'any', QFunction(0)) + test_reading_one(b'save', QFunction(0)) + test_reading_one(b'raze', QFunction(0)) + test_reading_one(b'sums', QFunction(0)) + test_reading_one(b'prev', QFunction(0)) + + test_reading_one(b'(enlist `a)!(enlist 1)', QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), + qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST))) + test_reading_one(b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), + qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))) + test_reading_one(b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), + qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))) + test_reading_one(b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], + qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))) + test_reading_one(b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), b'4'], + [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), b'456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])) + test_reading_one(b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), + [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [b'a', numpy.int64(2)]), numpy.float32(5.5)])) + + test_reading_one(b'flip `abc`def!(1 2 3; 4 5 6)', qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)])) + test_reading_one(b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', + qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)])) + test_reading_one(b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + b"a c"])) + test_reading_one(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', + qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + [b"Arthur Dent", b"Zaphod Beeblebrox", b"Ford Prefect"]])) + test_reading_one(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; " "; "Ford Prefect"))', + qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + [b"Arthur Dent", b" ", b"Ford Prefect"]])) + test_reading_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), + qlist(numpy.array([3, 4]), qtype = QLONG_LIST), + qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]])) + test_reading_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), + qlist(numpy.array([3, 4]), qtype = QLONG_LIST), + qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])) + test_reading_one(b'1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b" "])) + test_reading_one(b'-1#([] sym:`x`x`x;str:" a")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b"a"])) + test_reading_one(b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b" "])) + test_reading_one(b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b"aa"])) + test_reading_one(b'([] name:`symbol$(); iq:`int$())', qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), + qlist(numpy.array([]), qtype = QINT_LIST)])) + test_reading_one(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)])) + test_reading_one(b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), + qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) + ) def test_reading_numpy_temporals(): BINARY = OrderedDict() @@ -354,8 +318,7 @@ def test_reading_numpy_temporals(): BINARY[query] = binary - print('Deserialization (numpy temporals)') - for query, value in iter(NUMPY_TEMPORAL_EXPRESSIONS.items()): + def test_reading_numpy_temporals_one(query,value): buffer_ = BytesIO() binary = binascii.unhexlify(BINARY[query]) @@ -377,6 +340,35 @@ def test_reading_numpy_temporals(): print('.') + print('Deserialization (numpy temporals)') + test_reading_numpy_temporals_one(b'(2001.01m; 0Nm)', qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST)) + test_reading_numpy_temporals_one(b'2001.01m', numpy.datetime64('2001-01', 'M')) + test_reading_numpy_temporals_one(b'0Nm', numpy.datetime64('NaT', 'M')) + test_reading_numpy_temporals_one(b'2001.01.01 2000.05.01 0Nd', qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST)) + test_reading_numpy_temporals_one(b'2001.01.01', numpy.datetime64('2001-01-01', 'D')) + test_reading_numpy_temporals_one(b'0Nd', numpy.datetime64('NaT', 'D')) + test_reading_numpy_temporals_one(b'2000.01.04T05:36:57.600 0Nz', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST)) + test_reading_numpy_temporals_one(b'2000.01.04T05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600', 'ms')) + test_reading_numpy_temporals_one(b'0Nz', numpy.datetime64('NaT', 'ms')) + test_reading_numpy_temporals_one(b'12:01 0Nu', qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE)) + test_reading_numpy_temporals_one(b'12:01', numpy.timedelta64(721, 'm')) + test_reading_numpy_temporals_one(b'0Nu', numpy.timedelta64('NaT', 'm')) + test_reading_numpy_temporals_one(b'12:05:00 0Nv', qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND)) + test_reading_numpy_temporals_one(b'12:05:00', numpy.timedelta64(43500, 's')) + test_reading_numpy_temporals_one(b'0Nv', numpy.timedelta64('nat', 's')) + test_reading_numpy_temporals_one(b'12:04:59.123 0Nt', qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST)) + test_reading_numpy_temporals_one(b'12:04:59.123', numpy.timedelta64(43499123, 'ms')) + test_reading_numpy_temporals_one(b'0Nt', numpy.timedelta64('NaT', 'ms')) + test_reading_numpy_temporals_one(b'2000.01.04D05:36:57.600 0Np', qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST)) + test_reading_numpy_temporals_one(b'2000.01.04D05:36:57.600', numpy.datetime64('2000-01-04T05:36:57.600', 'ns')) + test_reading_numpy_temporals_one(b'0Np', numpy.datetime64('NaT', 'ns')) + test_reading_numpy_temporals_one(b'0D05:36:57.600 0Nn', qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST)) + test_reading_numpy_temporals_one(b'0D05:36:57.600', numpy.timedelta64(20217600000000, 'ns')) + test_reading_numpy_temporals_one(b'0Nn', numpy.timedelta64('NaT', 'ns')) + test_reading_numpy_temporals_one(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + qtable(['pos', 'dates'], + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')])) def test_reading_compressed(): BINARY = OrderedDict() @@ -408,7 +400,7 @@ def test_reading_compressed(): assert compare(value, result), 'deserialization failed: %s' % (query) header = buffer_reader.read_header(source = buffer_.getvalue()) - result = buffer_reader.read_data(message_size = header.size, is_compressed = header.is_compressed) + result = buffer_reader.read_data(message_size = header.size, compression_mode = header.compression_mode) assert compare(value, result), 'deserialization failed: %s' % (query) stream_reader = qreader.QReader(buffer_) diff --git a/tests/qtypes_test.py b/tests/qtypes_test.py index bb1ce92..d70aae1 100644 --- a/tests/qtypes_test.py +++ b/tests/qtypes_test.py @@ -177,6 +177,9 @@ def test_qtemporallist(): assert t[x].raw == na_dt[x] x += 1 + monthList = qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST) + assert str(monthList) == '[2001-01, NaT]' + assert repr(monthList) == 'QTemporalList([2001-01, NaT], qtype=-13)' def test_array_to_raw_qtemporal(): na_dt = numpy.arange('1999-01', '2005-12', dtype='datetime64[M]') @@ -195,7 +198,7 @@ def test_array_to_raw_qtemporal(): assert na[x] == x - 365 x += 1 - na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 12345678, dtype='datetime64[ms]') + na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 12345678, dtype='datetime64[ms]') na = array_to_raw_qtemporal(na_dt, qtype=QDATETIME_LIST) assert na.dtype == numpy.float64 @@ -208,7 +211,7 @@ def test_array_to_raw_qtemporal(): ref = (x * step) - 365 assert abs(na[x] - ref) < 0.1, '%s %s' %(na[x], ref) - na_dt = numpy.arange('1999-01-01T00:00:00.000Z', '2001-01-04T05:36:57.600Z', 1234567890000, dtype='datetime64[ns]') + na_dt = numpy.arange('1999-01-01T00:00:00.000', '2001-01-04T05:36:57.600', 1234567890000, dtype='datetime64[ns]') na = array_to_raw_qtemporal(na_dt, qtype=QTIMESTAMP_LIST) assert na.dtype == numpy.int64 @@ -249,7 +252,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[M]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'M'): + if not numpy.isnan(na_dt[x]): assert na_dt[x].astype(int) == raw[x] + 360 else: assert raw[x] == qnull(QMONTH) @@ -259,7 +262,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[D]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'D'): + if not numpy.isnan(na_dt[x]): assert na_dt[x].astype(int) == raw[x] + 10957 else: assert raw[x] == qnull(QDATE) @@ -269,7 +272,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[m]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 'm'): + if not numpy.isnan(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QMINUTE) @@ -279,7 +282,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[s]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 's'): + if not numpy.isnan(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QSECOND) @@ -289,7 +292,7 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('timedelta64[ms]') for x in range(len(na_dt)): - if na_dt[x] != numpy.timedelta64('NaT', 'ms'): + if not numpy.isnan(na_dt[x]): assert na_dt[x].astype(int) == raw[x] else: assert raw[x] == qnull(QTIME) @@ -309,19 +312,22 @@ def test_array_from_raw_qtemporal(): assert str(na_dt.dtype).startswith('datetime64[ns]') for x in range(len(na_dt)): - if na_dt[x] != numpy.datetime64('NaT', 'ns'): - assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00Z', 'ns').astype(numpy.int64) + if not numpy.isnan(na_dt[x]): + assert na_dt[x].astype(numpy.int64) == raw[x] + numpy.datetime64('2000-01-01T00:00:00', 'ns').astype(numpy.int64) else: assert raw[x] == qnull(QTIMESTAMP) raw = numpy.array([3.234, qnull(QDATETIME)]) na_dt = array_from_raw_qtemporal(raw, qtype=QDATETIME) - ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]) + ref = numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]) assert str(na_dt.dtype).startswith('datetime64[ms]') for x in range(len(na_dt)): - assert na_dt[x] == ref[x] + if not numpy.isnan(na_dt[x]): + assert na_dt[x] == ref[x] + else: + assert numpy.isnan(ref[x]) test_is_null() diff --git a/tests/qwriter_test.py b/tests/qwriter_test.py index 1ebaee0..24f974e 100644 --- a/tests/qwriter_test.py +++ b/tests/qwriter_test.py @@ -27,319 +27,6 @@ BINARY = OrderedDict() -EXPRESSIONS = OrderedDict(( - (b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', - qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)), - (b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')), - (b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')), - (b'(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST), - qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST), - qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST), - qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST), - numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), - )), - (b'2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH), - numpy.datetime64('2001-01', 'M'))), - (b'0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH), - qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH), - numpy.datetime64('NaT', 'M'))), - (b'2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01-01', 'D'), numpy.datetime64('2000-05-01', 'D'), numpy.datetime64('NaT', 'D')]), qtype = QDATE_LIST), qtype = QDATE_LIST), - qlist([366, 121, qnull(QDATE)], qtype=QDATE_LIST), - qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST), - numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), - )), - (b'2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), - numpy.datetime64('2001-01-01', 'D'))), - (b'0Nd', (qtemporal(qnull(QDATE), qtype=QDATE), - qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE), - numpy.datetime64('NaT', 'D'))), - (b'2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST), - qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST), - qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST), - numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), numpy.datetime64('nat', 'ms')]) - )), - (b'2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'), qtype=QDATETIME), - numpy.datetime64('2000-01-04T05:36:57.600Z', 'ms'))), - (b'0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME), - qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME), - numpy.datetime64('NaT', 'ms'))), - (b'12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype=QMINUTE_LIST), qtype=QMINUTE_LIST), - qlist([721, qnull(QMINUTE)], qtype=QMINUTE_LIST), - qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE), - numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), - )), - (b'12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE), - numpy.timedelta64(721, 'm'))), - (b'0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE), - qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE), - numpy.timedelta64('NaT', 'm'))), - (b'12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype=QSECOND_LIST), qtype=QSECOND_LIST), - qlist([43500, qnull(QSECOND)], qtype=QSECOND_LIST), - qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND), - numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]) - )), - (b'12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND), - numpy.timedelta64(43500, 's'))), - (b'0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND), - qtemporal(numpy.timedelta64('nat', 's'), qtype=QSECOND), - numpy.timedelta64('nat', 's'))), - (b'12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST), - qlist([43499123, qnull(QTIME)], qtype=QTIME_LIST), - qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST), - numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]) - )), - (b'12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME), - numpy.timedelta64(43499123, 'ms'))), - (b'0Nt', (qtemporal(qnull(QTIME), qtype=QTIME), - qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME), - numpy.timedelta64('NaT', 'ms'))), - (b'2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST), - qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST), - qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST), - numpy.array([numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), numpy.datetime64('nat', 'ns')]) - )), - (b'2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'), qtype=QTIMESTAMP), - numpy.datetime64('2000-01-04T05:36:57.600Z', 'ns'))), - (b'0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP), - qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP), - numpy.datetime64('NaT', 'ns'))), - (b'0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST), - qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype=QTIMESPAN_LIST), qtype=QTIMESPAN_LIST), - qlist([long(20217600000000), qnull(QTIMESPAN)], qtype=QTIMESPAN_LIST), - qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST), - numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]) - )), - (b'0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN), - numpy.timedelta64(20217600000000, 'ns'))), - (b'0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN), - qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN), - numpy.timedelta64('NaT', 'ns'))), - - (b'::', None), - (b'1+`', QException('type')), - (b'1', numpy.int64(1)), - (b'1i', numpy.int32(1)), - (b'-234h', numpy.int16(-234)), - (b'0b', numpy.bool_(False)), - (b'1b', numpy.bool_(True)), - (b'0x2a', numpy.byte(0x2a)), - (b'89421099511627575j', numpy.int64(long(89421099511627575))), - (b'5.5e', numpy.float32(5.5)), - (b'3.234', numpy.float64(3.234)), - (b'"0"', '0'), - (b'"abc"', ('abc', - numpy.array(list('abc'), dtype='S'))), - (b'"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog'), - (b'`abc', numpy.string_('abc')), - (b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')), - (b'0Nh', qnull(QSHORT)), - (b'0N', qnull(QLONG)), - (b'0Ni', qnull(QINT)), - (b'0Nj', qnull(QLONG)), - (b'0Ne', qnull(QFLOAT)), - (b'0n', qnull(QDOUBLE)), - (b'" "', qnull(QSTRING)), - (b'`', qnull(QSYMBOL)), - (b'0Ng', qnull(QGUID)), - (b'()', []), - (b'(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_), - qlist(numpy.array([False, True, False]), qtype = QBOOL_LIST), - qlist([False, True, False], qtype = QBOOL_LIST))), - (b'(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), - qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST), - qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST), - qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST))), - (b'(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16), - qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype = QSHORT_LIST), - qlist(numpy.array([1, 2, 3]), qtype = QSHORT_LIST), - qlist([1, 2, 3], qtype = QSHORT_LIST))), - (b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)), - (b'1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64), - qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), - qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist([1, 2, 3], qtype = QLONG_LIST))), - (b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32), - qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype = QINT_LIST), - qlist(numpy.array([1, 2, 3]), qtype = QINT_LIST), - qlist([1, 2, 3], qtype = QINT_LIST))), - (b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)), - (b'(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64), - qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), - qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist([1, 2, 3], qtype = QLONG_LIST))), - (b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)), - (b'(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32), - qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype = QFLOAT_LIST), - qlist(numpy.array([5.5, 8.5]), qtype = QFLOAT_LIST), - qlist([5.5, 8.5], qtype = QFLOAT_LIST))), - (b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)), - (b'3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64), - qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype = QDOUBLE_LIST), - qlist(numpy.array([3.23, 6.46]), qtype = QDOUBLE_LIST), - qlist([3.23, 6.46], qtype = QDOUBLE_LIST))), - (b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)), - (b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]), - (b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]), - (b'(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']), - (b'(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]), - - (b'`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_), - qlist(numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), qtype = QSYMBOL_LIST), - qlist(['jumps', 'over', 'a', 'lazy', 'dog'], qtype = QSYMBOL_LIST))), - (b'`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=numpy.object)), - (b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=numpy.object), qtype=QSYMBOL_LIST)), - (b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=numpy.object), qtype=QSYMBOL_LIST)), - (b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', - (['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], - qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST), - qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST))), - (b'{x+y}', QLambda('{x+y}')), - (b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])), - - (b'(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), - qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)), - QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), - qlist(numpy.array([1]), qtype=QLONG_LIST)))), - (b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), - qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))), - (b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), - qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))), - (b'(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), - [numpy.string_('a'), numpy.int64(2)])), - (b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], - qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))), - (b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], - [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])), - (b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), - [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])), - - (b'flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST), - qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)], - qtype=QTABLE), - qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]), - qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST), - [qlist([1, 2, 3], qtype = QLONG_LIST), - qlist([4, 5, 6], qtype = QLONG_LIST)]), - qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST), - [qlist([1, 2, 3]), qlist([4, 5, 6])], - **{'abc': QLONG_LIST, 'def': QLONG_LIST}), - qtable(['abc', 'def'], - [[1, 2, 3], [4, 5, 6]], - **{'abc': QLONG, 'def': QLONG}))), - (b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', - (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126], dtype=numpy.int64), qtype = QLONG_LIST)]), - qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)]), - qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), - [qlist(['Dent', 'Beeblebrox', 'Prefect'], qtype = QSYMBOL_LIST), - qlist([98, 42, 126], qtype = QLONG_LIST)]), - qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), - [qlist(['Dent', 'Beeblebrox', 'Prefect']), - qlist([98, 42, 126])], - name = QSYMBOL, iq = QLONG), - qtable(['name', 'iq'], - [['Dent', 'Beeblebrox', 'Prefect'], - [98, 42, 126]], - name = QSYMBOL, iq = QLONG), - qtable(['name', 'iq'], - [['Dent', 'Beeblebrox', 'Prefect'], - [98, 42, 126]], - **{'name': QSYMBOL, 'iq': QLONG}))), - (b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', - qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - "a c"])), - (b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', - qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - qlist(numpy.array(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]), qtype = QSTRING_LIST)])), - (b'flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))', - qtable(qlist(numpy.array(['name', 'iq', 'misc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), - qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), - qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", long(160), qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])), - (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), - [qlist(numpy.array([1, 2], dtype=numpy.int64), qtype = QLONG_LIST), - qlist(numpy.array([3, 4], dtype=numpy.int64), qtype = QLONG_LIST), - qlist(numpy.array([5, 6, 7], dtype=numpy.int64), qtype = QLONG_LIST)]]), - qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), - qlist(numpy.array([3, 4]), qtype = QLONG_LIST), - qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]]), - qtable(qlist(['sc', 'nsc'], qtype = QSYMBOL_LIST), - [qlist([1, 2, 3], qtype = QLONG_LIST), - [qlist([1, 2], qtype = QLONG_LIST), - qlist([3, 4], qtype = QLONG_LIST), - qlist([5, 6, 7], qtype = QLONG_LIST)]]))), - (b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), - [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), - qlist(numpy.array([3, 4]), qtype = QLONG_LIST), - qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])), - (b'1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b" "]), - 'single_char_strings': True - }), - (b'-1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b"a"]), - 'single_char_strings': True - }), - (b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b" "])), - (b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), - b"aa"])), - (b'([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), - qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]), - qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([]), qtype = QSYMBOL_LIST), - qlist(numpy.array([]), qtype = QINT_LIST)]), - qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), - [qlist([], qtype = QSYMBOL_LIST), - qlist([], qtype = QINT_LIST)]))), - (b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - (qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]), - qtable(['pos', 'dates'], - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')]) - )), - (b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', - QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), - [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), - qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), - [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), - qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) - ), - )) - - - def init(): with open('tests/QExpressions3.out', 'rb') as f: while True: @@ -356,7 +43,7 @@ def init(): def test_writing(): w = qwriter.QWriter(None, 3) - for query, value in iter(EXPRESSIONS.items()): + def test_writing_one(query, value): sys.stdout.write( '%-75s' % query ) if isinstance(value, tuple): for obj in value: @@ -375,6 +62,316 @@ def test_writing(): print('') + test_writing_one(b'("G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"; 0Ng)', + qlist(numpy.array([uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661'), qnull(QGUID)]), qtype=QGUID_LIST)) + test_writing_one(b'"G"$"8c680a01-5a49-5aab-5a65-d4bfddb6a661"', uuid.UUID('8c680a01-5a49-5aab-5a65-d4bfddb6a661')) + test_writing_one(b'"G"$"00000000-0000-0000-0000-000000000000"', uuid.UUID('00000000-0000-0000-0000-000000000000')) + test_writing_one (b'(2001.01m; 0Nm)', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01', 'M'), QMONTH), qnull(QMONTH)]), qtype=QMONTH_LIST), + qlist(numpy.array([12, qnull(QMONTH)]), qtype=QMONTH_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01', 'M'), numpy.datetime64('NaT', 'M')]), qtype = QMONTH_LIST), qtype = QMONTH_LIST), + qlist([12, qnull(QMONTH)], qtype=QMONTH_LIST), + qlist(numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), qtype=QMONTH_LIST), + numpy.array([numpy.datetime64('2001-01'), numpy.datetime64('NaT')], dtype='datetime64[M]'), + )) + test_writing_one(b'2001.01m', (qtemporal(numpy.datetime64('2001-01', 'M'), qtype=QMONTH), + numpy.datetime64('2001-01', 'M'))) + test_writing_one(b'0Nm', (qtemporal(qnull(QMONTH), qtype=QMONTH), + qtemporal(numpy.datetime64('NaT', 'M'), qtype=QMONTH), + numpy.datetime64('NaT', 'M'))) + test_writing_one(b'2001.01.01 2000.05.01 0Nd', (qlist(numpy.array([to_raw_qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), to_raw_qtemporal(numpy.datetime64('2000-05-01', 'D'), qtype=QDATE), qnull(QDATE)]), qtype=QDATE_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2001-01-01', 'D'), numpy.datetime64('2000-05-01', 'D'), numpy.datetime64('NaT', 'D')]), qtype = QDATE_LIST), qtype = QDATE_LIST), + qlist([366, 121, qnull(QDATE)], qtype=QDATE_LIST), + qlist(numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), qtype=QDATE_LIST), + numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]'), + )) + test_writing_one(b'2001.01.01', (qtemporal(numpy.datetime64('2001-01-01', 'D'), qtype=QDATE), + numpy.datetime64('2001-01-01', 'D'))) + test_writing_one(b'0Nd', (qtemporal(qnull(QDATE), qtype=QDATE), + qtemporal(numpy.datetime64('NaT', 'D'), qtype=QDATE), + numpy.datetime64('NaT', 'D'))) + test_writing_one(b'2000.01.04T05:36:57.600 0Nz', (qlist(numpy.array([3.234, qnull(QDATETIME)]), qtype=QDATETIME_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype=QDATETIME_LIST), qtype=QDATETIME_LIST), + qlist([3.234, qnull(QDATETIME)], qtype=QDATETIME_LIST), + qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]), qtype = QDATETIME_LIST), + numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), numpy.datetime64('nat', 'ms')]) + )) + test_writing_one(b'2000.01.04T05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ms'), qtype=QDATETIME), + numpy.datetime64('2000-01-04T05:36:57.600', 'ms'))) + test_writing_one(b'0Nz', (qtemporal(qnull(QDATETIME), qtype=QDATETIME), + qtemporal(numpy.datetime64('NaT', 'ms'), qtype=QDATETIME), + numpy.datetime64('NaT', 'ms'))) + test_writing_one(b'12:01 0Nu', (qlist(numpy.array([721, qnull(QMINUTE)]), qtype=QMINUTE_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype=QMINUTE_LIST), qtype=QMINUTE_LIST), + qlist([721, qnull(QMINUTE)], qtype=QMINUTE_LIST), + qlist(numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), qtype = QMINUTE), + numpy.array([numpy.timedelta64(721, 'm'), numpy.timedelta64('nat', 'm')]), + )) + test_writing_one(b'12:01', (qtemporal(numpy.timedelta64(721, 'm'), qtype=QMINUTE), + numpy.timedelta64(721, 'm'))) + test_writing_one(b'0Nu', (qtemporal(qnull(QMINUTE), qtype=QMINUTE), + qtemporal(numpy.timedelta64('NaT', 'm'), qtype=QMINUTE), + numpy.timedelta64('NaT', 'm'))) + test_writing_one(b'12:05:00 0Nv', (qlist(numpy.array([43500, qnull(QSECOND)]), qtype=QSECOND_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype=QSECOND_LIST), qtype=QSECOND_LIST), + qlist([43500, qnull(QSECOND)], qtype=QSECOND_LIST), + qlist(numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]), qtype = QSECOND), + numpy.array([numpy.timedelta64(43500, 's'), numpy.timedelta64('nat', 's')]) + )) + test_writing_one(b'12:05:00', (qtemporal(numpy.timedelta64(43500, 's'), qtype=QSECOND), + numpy.timedelta64(43500, 's'))) + test_writing_one(b'0Nv', (qtemporal(qnull(QSECOND), qtype=QSECOND), + qtemporal(numpy.timedelta64('nat', 's'), qtype=QSECOND), + numpy.timedelta64('nat', 's'))) + test_writing_one(b'12:04:59.123 0Nt', (qlist(numpy.array([43499123, qnull(QTIME)]), qtype=QTIME_LIST), + qlist([43499123, qnull(QTIME)], qtype=QTIME_LIST), + qlist(numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]), qtype = QTIME_LIST), + numpy.array([numpy.timedelta64(43499123, 'ms'), numpy.timedelta64('nat', 'ms')]) + )) + test_writing_one(b'12:04:59.123', (qtemporal(numpy.timedelta64(43499123, 'ms'), qtype=QTIME), + numpy.timedelta64(43499123, 'ms'))) + test_writing_one(b'0Nt', (qtemporal(qnull(QTIME), qtype=QTIME), + qtemporal(numpy.timedelta64('NaT', 'ms'), qtype=QTIME), + numpy.timedelta64('NaT', 'ms'))) + test_writing_one(b'2000.01.04D05:36:57.600 0Np', (qlist(numpy.array([long(279417600000000), qnull(QTIMESTAMP)]), qtype=QTIMESTAMP_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype=QTIMESTAMP_LIST), qtype=QTIMESTAMP_LIST), + qlist([long(279417600000000), qnull(QTIMESTAMP)], qtype=QTIMESTAMP_LIST), + qlist(numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]), qtype = QTIMESTAMP_LIST), + numpy.array([numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), numpy.datetime64('nat', 'ns')]) + )) + test_writing_one(b'2000.01.04D05:36:57.600', (qtemporal(numpy.datetime64('2000-01-04T05:36:57.600', 'ns'), qtype=QTIMESTAMP), + numpy.datetime64('2000-01-04T05:36:57.600', 'ns'))) + test_writing_one(b'0Np', (qtemporal(qnull(QTIMESTAMP), qtype=QTIMESTAMP), + qtemporal(numpy.datetime64('NaT', 'ns'), qtype=QTIMESTAMP), + numpy.datetime64('NaT', 'ns'))) + test_writing_one(b'0D05:36:57.600 0Nn', (qlist(numpy.array([long(20217600000000), qnull(QTIMESPAN)]), qtype=QTIMESPAN_LIST), + qlist(array_to_raw_qtemporal(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype=QTIMESPAN_LIST), qtype=QTIMESPAN_LIST), + qlist([long(20217600000000), qnull(QTIMESPAN)], qtype=QTIMESPAN_LIST), + qlist(numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]), qtype = QTIMESPAN_LIST), + numpy.array([numpy.timedelta64(20217600000000, 'ns'), numpy.timedelta64('nat', 'ns')]) + )) + test_writing_one(b'0D05:36:57.600', (qtemporal(numpy.timedelta64(20217600000000, 'ns'), qtype=QTIMESPAN), + numpy.timedelta64(20217600000000, 'ns'))) + test_writing_one(b'0Nn', (qtemporal(qnull(QTIMESPAN), qtype=QTIMESPAN), + qtemporal(numpy.timedelta64('NaT', 'ns'), qtype=QTIMESPAN), + numpy.timedelta64('NaT', 'ns'))) + + test_writing_one(b'::', None) + test_writing_one(b'1+`', QException('type')) + test_writing_one(b'1', numpy.int64(1)) + test_writing_one(b'1i', numpy.int32(1)) + test_writing_one(b'-234h', numpy.int16(-234)) + test_writing_one(b'0b', numpy.bool_(False)) + test_writing_one(b'1b', numpy.bool_(True)) + test_writing_one(b'0x2a', numpy.byte(0x2a)) + test_writing_one(b'89421099511627575j', numpy.int64(long(89421099511627575))) + test_writing_one(b'5.5e', numpy.float32(5.5)) + test_writing_one(b'3.234', numpy.float64(3.234)) + test_writing_one(b'"0"', '0') + test_writing_one(b'"abc"', ('abc', + numpy.array(list('abc'), dtype='S'))) + test_writing_one(b'"quick brown fox jumps over a lazy dog"', 'quick brown fox jumps over a lazy dog') + test_writing_one(b'`abc', numpy.string_('abc')) + test_writing_one(b'`quickbrownfoxjumpsoveralazydog', numpy.string_('quickbrownfoxjumpsoveralazydog')) + test_writing_one(b'0Nh', qnull(QSHORT)) + test_writing_one(b'0N', qnull(QLONG)) + test_writing_one(b'0Ni', qnull(QINT)) + test_writing_one(b'0Nj', qnull(QLONG)) + test_writing_one(b'0Ne', qnull(QFLOAT)) + test_writing_one(b'0n', qnull(QDOUBLE)) + test_writing_one(b'" "', qnull(QSTRING)) + test_writing_one(b'`', qnull(QSYMBOL)) + test_writing_one(b'0Ng', qnull(QGUID)) + test_writing_one(b'()', []) + test_writing_one(b'(0b;1b;0b)', (numpy.array([False, True, False], dtype=numpy.bool_), + qlist(numpy.array([False, True, False]), qtype = QBOOL_LIST), + qlist([False, True, False], qtype = QBOOL_LIST))) + test_writing_one(b'(0x01;0x02;0xff)', (numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), + qlist(numpy.array([0x01, 0x02, 0xff], dtype=numpy.byte), qtype = QBYTE_LIST), + qlist(numpy.array([0x01, 0x02, 0xff]), qtype = QBYTE_LIST), + qlist([0x01, 0x02, 0xff], qtype = QBYTE_LIST))) + test_writing_one(b'(1h;2h;3h)', (numpy.array([1, 2, 3], dtype=numpy.int16), + qlist(numpy.array([1, 2, 3], dtype=numpy.int16), qtype = QSHORT_LIST), + qlist(numpy.array([1, 2, 3]), qtype = QSHORT_LIST), + qlist([1, 2, 3], qtype = QSHORT_LIST))) + test_writing_one(b'(1h;0Nh;3h)', qlist(numpy.array([1, qnull(QSHORT), 3], dtype=numpy.int16), qtype=QSHORT_LIST)) + test_writing_one(b'1 2 3', (numpy.array([1, 2, 3], dtype=numpy.int64), + qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), + qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist([1, 2, 3], qtype = QLONG_LIST))) + test_writing_one(b'1 0N 3', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_writing_one(b'(1i;2i;3i)', (numpy.array([1, 2, 3], dtype=numpy.int32), + qlist(numpy.array([1, 2, 3], dtype=numpy.int32), qtype = QINT_LIST), + qlist(numpy.array([1, 2, 3]), qtype = QINT_LIST), + qlist([1, 2, 3], qtype = QINT_LIST))) + test_writing_one(b'(1i;0Ni;3i)', qlist(numpy.array([1, qnull(QINT), 3], dtype=numpy.int32), qtype=QINT_LIST)) + test_writing_one(b'(1j;2j;3j)', (numpy.array([1, 2, 3], dtype=numpy.int64), + qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), + qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist([1, 2, 3], qtype = QLONG_LIST))) + test_writing_one(b'(1j;0Nj;3j)', qlist(numpy.array([1, qnull(QLONG), 3], dtype=numpy.int64), qtype=QLONG_LIST)) + test_writing_one(b'(5.5e; 8.5e)', (numpy.array([5.5, 8.5], dtype=numpy.float32), + qlist(numpy.array([5.5, 8.5], dtype=numpy.float32), qtype = QFLOAT_LIST), + qlist(numpy.array([5.5, 8.5]), qtype = QFLOAT_LIST), + qlist([5.5, 8.5], qtype = QFLOAT_LIST))) + test_writing_one(b'(5.5e; 0Ne)', qlist(numpy.array([5.5, qnull(QFLOAT)], dtype=numpy.float32), qtype=QFLOAT_LIST)) + test_writing_one(b'3.23 6.46', (numpy.array([3.23, 6.46], dtype=numpy.float64), + qlist(numpy.array([3.23, 6.46], dtype=numpy.float64), qtype = QDOUBLE_LIST), + qlist(numpy.array([3.23, 6.46]), qtype = QDOUBLE_LIST), + qlist([3.23, 6.46], qtype = QDOUBLE_LIST))) + test_writing_one(b'3.23 0n', qlist(numpy.array([3.23, qnull(QDOUBLE)], dtype=numpy.float64), qtype=QDOUBLE_LIST)) + test_writing_one(b'(1;`bcd;"0bc";5.5e)', [numpy.int64(1), numpy.string_('bcd'), '0bc', numpy.float32(5.5)]) + test_writing_one(b'(42;::;`foo)', [numpy.int64(42), None, numpy.string_('foo')]) + test_writing_one(b'(1;2h;3.234;"4")', [numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4']) + test_writing_one(b'(`one;2 3;"456";(7;8 9))', [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]]) + + test_writing_one(b'`jumps`over`a`lazy`dog', (numpy.array(['jumps', 'over', 'a', 'lazy', 'dog'], dtype=numpy.string_), + qlist(numpy.array(['jumps', 'over', 'a', 'lazy', 'dog']), qtype = QSYMBOL_LIST), + qlist(['jumps', 'over', 'a', 'lazy', 'dog'], qtype = QSYMBOL_LIST))) + test_writing_one(b'`the`quick`brown`fox', numpy.array([numpy.string_('the'), numpy.string_('quick'), numpy.string_('brown'), numpy.string_('fox')], dtype=object)) + test_writing_one(b'``quick``fox', qlist(numpy.array([qnull(QSYMBOL), numpy.string_('quick'), qnull(QSYMBOL), numpy.string_('fox')], dtype=object), qtype=QSYMBOL_LIST)) + test_writing_one(b'``', qlist(numpy.array([qnull(QSYMBOL), qnull(QSYMBOL)], dtype=object), qtype=QSYMBOL_LIST)) + test_writing_one(b'("quick"; "brown"; "fox"; "jumps"; "over"; "a lazy"; "dog")', + (['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], + qlist(numpy.array(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog']), qtype = QSTRING_LIST), + qlist(['quick', 'brown', 'fox', 'jumps', 'over', 'a lazy', 'dog'], qtype = QSTRING_LIST))) + test_writing_one(b'{x+y}', QLambda('{x+y}')) + test_writing_one(b'{x+y}[3]', QProjection([QLambda('{x+y}'), numpy.int64(3)])) + + test_writing_one(b'(enlist `a)!(enlist 1)', (QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), + qlist(numpy.array([1], dtype=numpy.int64), qtype=QLONG_LIST)), + QDictionary(qlist(numpy.array(['a']), qtype = QSYMBOL_LIST), + qlist(numpy.array([1]), qtype=QLONG_LIST)))) + test_writing_one(b'1 2!`abc`cdefgh', QDictionary(qlist(numpy.array([1, 2], dtype=numpy.int64), qtype=QLONG_LIST), + qlist(numpy.array(['abc', 'cdefgh']), qtype = QSYMBOL_LIST))) + test_writing_one(b'`abc`def`gh!([] one: 1 2 3; two: 4 5 6)', QDictionary(qlist(numpy.array(['abc', 'def', 'gh']), qtype = QSYMBOL_LIST), + qtable(qlist(numpy.array(['one', 'two']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]))) + test_writing_one(b'(`x`y!(`a;2))', QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), + [numpy.string_('a'), numpy.int64(2)])) + test_writing_one(b'(0 1; 2 3)!`first`second', QDictionary([qlist(numpy.array([0, 1], dtype=numpy.int64), qtype=QLONG_LIST), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST)], + qlist(numpy.array(['first', 'second']), qtype = QSYMBOL_LIST))) + test_writing_one(b'(1;2h;3.234;"4")!(`one;2 3;"456";(7;8 9))', QDictionary([numpy.int64(1), numpy.int16(2), numpy.float64(3.234), '4'], + [numpy.string_('one'), qlist(numpy.array([2, 3], dtype=numpy.int64), qtype=QLONG_LIST), '456', [numpy.int64(7), qlist(numpy.array([8, 9], dtype=numpy.int64), qtype=QLONG_LIST)]])) + test_writing_one(b'`A`B`C!((1;3.234;3);(`x`y!(`a;2));5.5e)', QDictionary(qlist(numpy.array(['A', 'B', 'C']), qtype = QSYMBOL_LIST), + [[numpy.int64(1), numpy.float64(3.234), numpy.int64(3)], QDictionary(qlist(numpy.array(['x', 'y']), qtype = QSYMBOL_LIST), [numpy.string_('a'), numpy.int64(2)]), numpy.float32(5.5)])) + + test_writing_one(b'flip `abc`def!(1 2 3; 4 5 6)', (qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype=QLONG_LIST), + qlist(numpy.array([4, 5, 6], dtype=numpy.int64), qtype=QLONG_LIST)], + qtype=QTABLE), + qtable(qlist(numpy.array(['abc', 'def']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + qlist(numpy.array([4, 5, 6]), qtype = QLONG_LIST)]), + qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST), + [qlist([1, 2, 3], qtype = QLONG_LIST), + qlist([4, 5, 6], qtype = QLONG_LIST)]), + qtable(qlist(['abc', 'def'], qtype = QSYMBOL_LIST), + [qlist([1, 2, 3]), qlist([4, 5, 6])], + **{'abc': QLONG_LIST, 'def': QLONG_LIST}), + qtable(['abc', 'def'], + [[1, 2, 3], [4, 5, 6]], + **{'abc': QLONG, 'def': QLONG}))) + test_writing_one(b'flip `name`iq!(`Dent`Beeblebrox`Prefect;98 42 126)', + (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126], dtype=numpy.int64), qtype = QLONG_LIST)]), + qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST)]), + qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), + [qlist(['Dent', 'Beeblebrox', 'Prefect'], qtype = QSYMBOL_LIST), + qlist([98, 42, 126], qtype = QLONG_LIST)]), + qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), + [qlist(['Dent', 'Beeblebrox', 'Prefect']), + qlist([98, 42, 126])], + name = QSYMBOL, iq = QLONG), + qtable(['name', 'iq'], + [['Dent', 'Beeblebrox', 'Prefect'], + [98, 42, 126]], + name = QSYMBOL, iq = QLONG), + qtable(['name', 'iq'], + [['Dent', 'Beeblebrox', 'Prefect'], + [98, 42, 126]], + **{'name': QSYMBOL, 'iq': QLONG}))) + test_writing_one(b'flip `name`iq`grade!(`Dent`Beeblebrox`Prefect;98 42 126;"a c")', + qtable(qlist(numpy.array(['name', 'iq', 'grade']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + "a c"])) + test_writing_one(b'flip `name`iq`fullname!(`Dent`Beeblebrox`Prefect;98 42 126;("Arthur Dent"; "Zaphod Beeblebrox"; "Ford Prefect"))', + qtable(qlist(numpy.array(['name', 'iq', 'fullname']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + qlist(numpy.array(["Arthur Dent", "Zaphod Beeblebrox", "Ford Prefect"]), qtype = QSTRING_LIST)])) + test_writing_one(b'flip `name`iq`misc!(`Dent`Beeblebrox`Prefect;98 42 126;("The Hitch Hiker\'s Guide to the Galaxy"; 160; 1979.10.12))', + qtable(qlist(numpy.array(['name', 'iq', 'misc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['Dent', 'Beeblebrox', 'Prefect']), qtype = QSYMBOL_LIST), + qlist(numpy.array([98, 42, 126]), qtype = QLONG_LIST), + qlist(numpy.array(["The Hitch Hiker\'s Guide to the Galaxy", long(160), qtemporal(numpy.datetime64('1979-10-12', 'D'), qtype=QDATE)]), qtype = QGENERAL_LIST)])) + test_writing_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6 7))', (qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3], dtype=numpy.int64), qtype = QLONG_LIST), + [qlist(numpy.array([1, 2], dtype=numpy.int64), qtype = QLONG_LIST), + qlist(numpy.array([3, 4], dtype=numpy.int64), qtype = QLONG_LIST), + qlist(numpy.array([5, 6, 7], dtype=numpy.int64), qtype = QLONG_LIST)]]), + qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), + qlist(numpy.array([3, 4]), qtype = QLONG_LIST), + qlist(numpy.array([5, 6, 7]), qtype = QLONG_LIST)]]), + qtable(qlist(['sc', 'nsc'], qtype = QSYMBOL_LIST), + [qlist([1, 2, 3], qtype = QLONG_LIST), + [qlist([1, 2], qtype = QLONG_LIST), + qlist([3, 4], qtype = QLONG_LIST), + qlist([5, 6, 7], qtype = QLONG_LIST)]]))) + test_writing_one(b'([] sc:1 2 3; nsc:(1 2; 3 4; 5 6))', qtable(qlist(numpy.array(['sc', 'nsc']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1, 2, 3]), qtype = QLONG_LIST), + [qlist(numpy.array([1, 2]), qtype = QLONG_LIST), + qlist(numpy.array([3, 4]), qtype = QLONG_LIST), + qlist(numpy.array([5, 6]), qtype = QLONG_LIST)]])) + test_writing_one(b'1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b" "]), + 'single_char_strings': True + }) + test_writing_one(b'-1#([] sym:`x`x`x;str:" a")', {'data': qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b"a"]), + 'single_char_strings': True + }) + test_writing_one(b'2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b" "])) + test_writing_one(b'-2#([] sym:`x`x`x`x;str:" aa")', qtable(qlist(numpy.array(['sym', 'str']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['x', 'x'], dtype=numpy.string_), qtype = QSYMBOL_LIST), + b"aa"])) + test_writing_one(b'([] name:`symbol$(); iq:`int$())', (qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([], dtype=numpy.string_), qtype = QSYMBOL_LIST), + qlist(numpy.array([], dtype=numpy.int32), qtype = QINT_LIST)]), + qtable(qlist(numpy.array(['name', 'iq']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([]), qtype = QSYMBOL_LIST), + qlist(numpy.array([]), qtype = QINT_LIST)]), + qtable(qlist(['name', 'iq'], qtype = QSYMBOL_LIST), + [qlist([], qtype = QSYMBOL_LIST), + qlist([], qtype = QINT_LIST)]))) + test_writing_one(b'([] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + (qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype=QDATE_LIST)]), + qtable(['pos', 'dates'], + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + numpy.array([numpy.datetime64('2001-01-01'), numpy.datetime64('2000-05-01'), numpy.datetime64('NaT')], dtype='datetime64[D]')]) + )) + test_writing_one(b'([eid:1001 1002 1003] pos:`d1`d2`d3;dates:(2001.01.01;2000.05.01;0Nd))', + QKeyedTable(qtable(qlist(numpy.array(['eid']), qtype = QSYMBOL_LIST), + [qlist(numpy.array([1001, 1002, 1003]), qtype = QLONG_LIST)]), + qtable(qlist(numpy.array(['pos', 'dates']), qtype = QSYMBOL_LIST), + [qlist(numpy.array(['d1', 'd2', 'd3']), qtype = QSYMBOL_LIST), + qlist(numpy.array([366, 121, qnull(QDATE)]), qtype = QDATE_LIST)])) + ) + + def test_write_single_char_string(): w = qwriter.QWriter(None, 3)