Merge branch 'release-v5.5.2'

Author: looselycoupled

.travis.yml

@@ -3,8 +3,6 @@ language: python
 dist: xenial
 
 python:
-  - '3.4'
-  - '3.5'
   - '3.6'
   - '3.7'
 

btrdb/conn.py

@@ -1,49 +1,46 @@
-# Copyright (c) 2017 Sam Kumar <samkumar@berkeley.edu>
-# Copyright (c) 2017 Michael P Andersen <m.andersen@cs.berkeley.edu>
-# Copyright (c) 2017 University of California, Berkeley
-# All rights reserved.
+# btrdb.conn
+# Connection related objects for the BTrDB library
 #
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright
-#       notice, this list of conditions and the following disclaimer in the
-#       documentation and/or other materials provided with the distribution.
-#     * Neither the name of the University of California, Berkeley nor the
-#       names of its contributors may be used to endorse or promote products
-#       derived from this software without specific prior written permission.
+# Author:   PingThings
+# Created:  Fri Dec 21 14:57:30 2018 -0500
 #
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNERS OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# For license information, see LICENSE.txt
+# ID: conn.py [] allen@pingthings.io $
 
 """
-The 'btrdb' module provides Python bindings to interact with BTrDB.
+Connection related objects for the BTrDB library
 """
 
-import grpc
-import uuid as uuidlib
+##########################################################################
+## Imports
+##########################################################################
+
 import os
+import re
+import json
+import uuid as uuidlib
 
+import grpc
 from grpc._cython.cygrpc import CompressionAlgorithm
 
 from btrdb.stream import Stream, StreamSet
 from btrdb.utils.general import unpack_stream_descriptor
 from btrdb.utils.conversion import to_uuid
+from btrdb.exceptions import NotFound
+
+##########################################################################
+## Module Variables
+##########################################################################
 
 MIN_TIME = -(16 << 56)
 MAX_TIME = 48 << 56
 MAX_POINTWIDTH = 63
 
 
+##########################################################################
+## Classes
+##########################################################################
+
 class Connection(object):
 
     def __init__(self, addrportstr, apikey=None):
@@ -106,6 +103,43 @@ class BTrDB(object):
     def __init__(self, endpoint):
         self.ep = endpoint
 
+    def query(self, stmt, params=[]):
+        """
+        Performs a SQL query on the database metadata and returns a list of
+        dictionaries from the resulting cursor.
+
+        Parameters
+        ----------
+        stmt: str
+            a SQL statement to be executed on the BTrDB metadata.  Available
+            tables are noted below.  To sanitize inputs use a `$1` style parameter such as
+            `select * from streams where name = $1 or name = $2`.
+        params: list or tuple
+            a list of parameter values to be sanitized and interpolated into the
+            SQL statement. Using parameters forces value/type checking and is
+            considered a best practice at the very least.
+
+        Returns
+        -------
+        list
+            a list of dictionary object representing the cursor results.
+
+
+        Notes
+        -------
+        Parameters will be inserted into the SQL statement as noted by the
+        paramter number such as `$1`, `$2`, or `$3`.  The `streams` table is
+        available for `SELECT` statements only.
+
+        See https://btrdb.readthedocs.io/en/latest/ for more info.
+        """
+        return [
+            json.loads(row.decode("utf-8"))
+            for page in self.ep.sql_query(stmt, params)
+            for row in page
+        ]
+
+
     def streams(self, *identifiers, versions=None):
         """
         Returns a StreamSet object with BTrDB streams from the supplied
@@ -129,7 +163,31 @@ def streams(self, *identifiers, versions=None):
         if versions and len(versions) != len(identifiers):
             raise ValueError("number of versions does not match identifiers")
 
-        streams = [self.stream_from_uuid(ident) for ident in identifiers]
+        streams = []
+        for ident in identifiers:
+            if isinstance(ident, uuidlib.UUID):
+                streams.append(self.stream_from_uuid(ident))
+                continue
+
+            if isinstance(ident, str):
+                # attempt UUID lookup
+                pattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+                if re.match(pattern, ident):
+                    streams.append(self.stream_from_uuid(ident))
+                    continue
+
+                # attempt collection/name lookup
+                if "/" in ident:
+                    parts = ident.split("/")
+                    found = self.streams_in_collection("/".join(parts[:-1]), tags={"name": parts[-1]})
+                    if len(found) == 1:
+                        streams.append(found[0])
+                        continue
+                    raise NotFound(f"Could not identify stream `{ident}`")
+
+            raise ValueError(f"Could not identify stream based on `{ident}`.  Identifier must be UUID or collection/name.")
+
+
         obj = StreamSet(streams)
 
         if versions:
@@ -249,6 +307,9 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non
         if annotations is None:
             annotations = {}
 
+        if not collection:
+            collection = [None]
+
         for item in collection:
             streams = self.ep.lookupStreams(item, is_collection_prefix, tags, annotations)
             for desclist in streams:

btrdb/endpoint.py

@@ -212,3 +212,9 @@ def generateCSV(self, queryType, start, end, width, depth, includeVersions, *str
         for result in self.stub.GenerateCSV(params):
             BTrDBError.checkProtoStat(result.stat)
             yield result.row
+
+    def sql_query(self, stmt, params=[]):
+        request = btrdb_pb2.SQLQueryParams(query=stmt, params=params)
+        for page in self.stub.SQLQuery(request):
+            BTrDBError.checkProtoStat(page.stat)
+            yield page.SQLQueryRow

btrdb/stream.py

@@ -89,9 +89,14 @@ def refresh_metadata(self):
         self._known_to_exist = True
 
         # deserialize annoation values
-        self._annotations = dict(
-            [[k, json.loads(v)] for k, v in self._annotations.items()]
-        )
+        parts = []
+        for k, v in self._annotations.items():
+            try:
+                parts.append([k, json.loads(v)])
+            except json.decoder.JSONDecodeError:
+                parts.append([k, v])
+
+        self._annotations = dict(parts)
 
     def exists(self):
         """
@@ -174,6 +179,21 @@ def name(self):
         """
         return self.tags()["name"]
 
+    @property
+    def unit(self):
+        """
+        Returns the stream's unit which is parsed from the stream tags.  This
+        may require a round trip to the server depending on how the stream was
+        acquired.
+
+        Returns
+        -------
+        str
+            The unit for values of the stream.
+
+        """
+        return self.tags()["unit"]
+
     @property
     def collection(self):
         """
@@ -921,7 +941,7 @@ def filter(self, start=None, end=None, collection=None, name=None, unit=None,
             # filters if the subset of the annotations matches the given annotations
             obj._streams = [
                 s for s in obj._streams
-                if annotations.items() <= s.annotations().items()
+                if annotations.items() <= s.annotations()[0].items()
             ]
 
         return obj

btrdb/utils/credentials.py

@@ -132,7 +132,9 @@ def credentials(endpoints=None, apikey=None):
     """
     creds = {}
     credentials_by_arg = filter_none(lambda: { "endpoints": endpoints, "apikey": apikey, })
-    pipeline = (credentials_by_profile, credentials_by_env, credentials_by_arg)
+    pipeline = [credentials_by_env, credentials_by_arg]
+    if os.path.exists(CREDENTIALS_PATH):
+        pipeline.insert(0, credentials_by_profile)
+
     [creds.update(func()) for func in pipeline]
     return creds
-

btrdb/version.py

@@ -18,9 +18,9 @@
 __version_info__ = {
     'major': 5,
     'minor': 5,
-    'micro': 1,
+    'micro': 2,
     'releaselevel': 'final',
-    'serial': 9,
+    'serial': 10,
 }
 
 ##########################################################################

docs/source/conf.py

@@ -49,6 +49,7 @@
     'sphinx.ext.napoleon',
     'sphinx.ext.todo',
     'sphinx.ext.githubpages',
+    'sphinx.ext.intersphinx',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/maintainers/anaconda.rst

@@ -3,7 +3,7 @@ Anaconda Package Creation Walkthrough
 
 This walkthrough documents the process to create and upload the btrdb package to Anaconda Cloud.  At a high level, this will guide you through the following steps:
 
-1. Activate your local environment and install dependenciess
+1. Activate your local environment and install dependencies
 2. Create and edit the `meta.yaml` file
 3. Create local system builds for 3.6 and 3.7
 4. Create builds for other platforms using conda's conversion command
@@ -117,27 +117,6 @@ This will create a `btrdb` subfolder containing the `meta.yaml` file we need.  A
     $ cd ../
     $ rm -rf conda.recipe/
 
-Modify the requirements file
------------------------------
-
-At this time, the codebase has pinned the grpc dependencies at `1.12.1` which needs to be modified by hand as that version doesnt exist in Anaconda.  A diff is provided below so you can edit or patch the file.
-
-  .. code-block:: diff
-
-    diff --git a/requirements.txt b/requirements.txt
-    index f504f26..403b1d4 100644
-    --- a/requirements.txt
-    +++ b/requirements.txt
-    @@ -1,6 +1,6 @@
-    # GRPC / Protobuff related
-    -grpcio==1.12.1
-    -grpcio-tools==1.12.1
-    +grpcio==1.16.1
-    +grpcio-tools==1.16.1
-
-    # Time related utils
-    pytz
-
 
 Modify the meta.yaml file
 --------------------------
@@ -169,19 +148,11 @@ You may wish to update the build number as well.  A patch is supplied below to g
 
     requirements:
     host:
-    -    - grpcio ==1.12.1
-    -    - grpcio-tools ==1.12.1
-    +    - grpcio ==1.16.1
-    +    - grpcio-tools ==1.16.1
         - pip
         - python
         - pytz
     +    - pytest-runner
     run:
-    -    - grpcio ==1.12.1
-    -    - grpcio-tools ==1.12.1
-    +    - grpcio ==1.16.1
-    +    - grpcio-tools ==1.16.1
         - python
         - pytz
     +    - pytest-runner

docs/source/working/stream-query-manage.rst

@@ -44,7 +44,7 @@ supplied UUID cannot be found then :code:`None` will be returned.
     stream = conn.stream_from_uuid("71466a91-dcfe-42ea-9e88-87c51f847942")
 
 
-Query by collection
+Finding by collection
 --------------------------
 You can also search for multiple streams by collection using the server object's
 :code:`streams_in_collection` method which will return a simple list of
@@ -56,3 +56,44 @@ detail.
 
     conn = btrdb.connect()
     streams = conn.streams_in_collection("NORTHEAST/NH")
+
+
+Querying Metadata
+-----------------
+Finally, you can query for metadata using standard SQL although at the moment, only the
+`streams` table is available.  SQL queries can be submitted using the `query`
+method which accepts both a `stmt` and `params` argument.  The `stmt` should
+contain the SQL you'd like executed with parameter placeholders such as `$1` or
+`$2` as shown below.
+
+.. code-block:: python
+
+    conn = btrdb.connect()
+    stmt = "select uuid from streams where name = $1 or name = $2"
+    params = ["Boston_1", "Boston_2"]
+
+    for row in conn.query(stmt, params):
+      print(row)
+
+The SQL query results are returned as a list of dictionaries where each key
+matches a column from your SQL projection.  You can choose your columns from the
+schema of the streams table as follows.
+
+
++------------------+------------------------+-----------+
+|      Column      |          Type          | Nullable  |
++==================+========================+===========+
+| uuid             | uuid                   | not null  |
++------------------+------------------------+-----------+
+| collection       | character varying(256) | not null  |
++------------------+------------------------+-----------+
+| name             | character varying(256) | not null  |
++------------------+------------------------+-----------+
+| unit             | character varying(256) | not null  |
++------------------+------------------------+-----------+
+| ingress          | character varying(256) | not null  |
++------------------+------------------------+-----------+
+| property_version | bigint                 | not null  |
++------------------+------------------------+-----------+
+| annotations      | hstore                 |           |
++------------------+------------------------+-----------+

setup.py

@@ -52,8 +52,6 @@
     'Natural Language :: English',
     'Operating System :: OS Independent',
     'Programming Language :: Python',
-    'Programming Language :: Python :: 3.4',
-    'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Topic :: Database',
@@ -153,7 +151,7 @@ def get_description_type(path=PKG_DESCRIBE):
         "console_scripts": [],
     },
     "install_requires": list(get_requires()),
-    "python_requires": ">=3.4, <4",
+    "python_requires": ">=3.6, <4",
     "setup_requires":["pytest-runner"],
     "tests_require":["pytest"],
 }