Merge branch 'release-5.5.1'

Author: bbengfort

btrdb/__init__.py

@@ -20,6 +20,7 @@
 from btrdb.exceptions import ConnectionError
 from btrdb.version import get_version
 from btrdb.utils.credentials import credentials_by_profile, credentials
+from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME
 
 ##########################################################################
 ## Module Variables
@@ -31,6 +32,7 @@
 BTRDB_API_KEY = "BTRDB_API_KEY"
 BTRDB_PROFILE = "BTRDB_PROFILE"
 
+
 ##########################################################################
 ## Functions
 ##########################################################################

btrdb/endpoint.py

@@ -42,7 +42,7 @@ def rawValues(self, uu, start, end, version = 0):
             yield result.values, result.versionMajor
 
     def alignedWindows(self, uu, start, end, pointwidth, version = 0):
-        params = btrdb_pb2.AlignedWindowsParams(uuid = uu.bytes, start = start, end = end, versionMajor = version, pointWidth = pointwidth)
+        params = btrdb_pb2.AlignedWindowsParams(uuid = uu.bytes, start = start, end = end, versionMajor = version, pointWidth = int(pointwidth))
         for result in self.stub.AlignedWindows(params):
             BTrDBError.checkProtoStat(result.stat)
             yield result.values, result.versionMajor

btrdb/point.py

@@ -85,6 +85,8 @@ def __str__(self):
         return self.__repr__()
 
     def __eq__(self, other):
+        if not hasattr(other, "time") or not hasattr(other, "value"):
+            return False
         return self.time == other.time and self.value == other.value
 
 
@@ -218,6 +220,10 @@ def __str__(self):
         return self.__repr__()
 
     def __eq__(self, other):
+        for attr in "time", "min", "mean", "max", "count", "stddev":
+            if not hasattr(other, attr):
+                return False
+
         return self.time == other.time and \
             self.min == other.min and \
             self.mean == other.mean and \

btrdb/stream.py

@@ -34,6 +34,8 @@
 ##########################################################################
 
 INSERT_BATCH_SIZE = 5000
+MINIMUM_TIME = -(16 << 56)
+MAXIMUM_TIME = (48 << 56) - 1
 
 try:
     RE_PATTERN = re._pattern_type
@@ -196,12 +198,14 @@ def collection(self):
 
     def earliest(self, version=0):
         """
-        Returns the first point of data in the stream.  Returns None if error
+        Returns the first point of data in the stream. Returns None if error
         encountered during lookup or no values in stream.
 
         Parameters
         ----------
-        None
+        version : int, default: 0
+            Specify the version of the stream to query; if zero, queries the latest
+            stream state rather than pinning to a version.
 
         Returns
         -------
@@ -210,18 +214,19 @@ def earliest(self, version=0):
             the value was retrieved at (tuple(RawPoint, int)).
 
         """
-        start = 0
+        start = MINIMUM_TIME
         return self.nearest(start, version=version, backward=False)
 
     def latest(self, version=0):
         """
-        Returns last point of data in the stream. Note that this method will
-        return None if no point can be found that is less than the current
-        date/time.
+        Returns last point of data in the stream. Returns None if error
+        encountered during lookup or no values in stream.
 
         Parameters
         ----------
-        None
+        version : int, default: 0
+            Specify the version of the stream to query; if zero, queries the latest
+            stream state rather than pinning to a version.
 
         Returns
         -------
@@ -230,9 +235,30 @@ def latest(self, version=0):
             the value was retrieved at (tuple(RawPoint, int)).
 
         """
-        start = currently_as_ns()
+        start = MAXIMUM_TIME
         return self.nearest(start, version=version, backward=True)
 
+    def current(self, version=0):
+        """
+        Returns the point that is closest to the current timestamp, e.g. the latest
+        point in the stream up until now. Note that no future values will be returned.
+        Returns None if errors during lookup or there are no values before now.
+
+        Parameters
+        ----------
+        version : int, default: 0
+            Specify the version of the stream to query; if zero, queries the latest
+            stream state rather than pinning to a version.
+
+        Returns
+        -------
+        tuple
+            The last data point in the stream up until now and the version of the stream
+            the value was retrieved at (tuple(RawPoint, int)).
+        """
+        start = currently_as_ns()
+        return self.nearest(start, version, backward=True)
+
     def tags(self, refresh=False):
         """
         Returns the stream's tags.
@@ -728,8 +754,7 @@ def versions(self):
 
     def earliest(self):
         """
-        Returns earliest points of data in streams using available
-        filters.
+        Returns earliest points of data in streams using available filters.
 
         Parameters
         ----------
@@ -743,7 +768,7 @@ def earliest(self):
         """
         earliest = []
         params = self._params_from_filters()
-        start = params.get("start", 0)
+        start = params.get("start", MINIMUM_TIME)
 
         for s in self._streams:
             version = self.versions()[s.uuid]
@@ -754,10 +779,7 @@ def earliest(self):
 
     def latest(self):
         """
-        Returns latest points of data in the streams using available
-        filters.  Note that this method will return None if no
-        end filter is provided and point cannot be found that is less than the
-        current date/time.
+        Returns latest points of data in the streams using available filters.
 
         Parameters
         ----------
@@ -771,7 +793,7 @@ def latest(self):
         """
         latest = []
         params = self._params_from_filters()
-        start = params.get("end", currently_as_ns())
+        start = params.get("end", MAXIMUM_TIME)
 
         for s in self._streams:
             version = self.versions()[s.uuid]
@@ -780,6 +802,37 @@ def latest(self):
 
         return tuple(latest)
 
+    def current(self):
+        """
+        Returns the points of data in the streams closest to the current timestamp. If
+        the current timestamp is outside of the filtered range of data, a ValueError is
+        raised.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        tuple
+            The latest points of data found among all streams
+        """
+        latest = []
+        params = self._params_from_filters()
+        now = currently_as_ns()
+        end = params.get("end", None)
+        start = params.get("start", None)
+
+        if (end is not None and end <= now) or (start is not None and start > now):
+            raise ValueError("current time is not included in filtered stream range")
+
+        for s in self._streams:
+            version = self.versions()[s.uuid]
+            point, _ = s.nearest(now, version=version, backward=True)
+            latest.append(point)
+
+        return tuple(latest)
+
     def filter(self, start=None, end=None, collection=None, name=None, unit=None,
                tags=None, annotations=None):
         """

btrdb/utils/general.py

@@ -30,3 +30,154 @@ def unpack_stream_descriptor(desc):
     for ann in desc.annotations:
         anns[ann.key] = ann.val.value
     return tags, anns
+
+
+##########################################################################
+## Pointwidth Helpers
+##########################################################################
+
+class pointwidth(object):
+    """
+    A representation of a period of time described by the BTrDB tree (and used in
+    aligned_windows queries). The pointwidth allows users to traverse to different
+    levels of the BTrDB tree and to select StatPoints directly, vastly improving the
+    performance of queries. However, because the real duration of the pointwidth is
+    defined in powers of 2 (e.g. 2**pointwidth nanoseconds), the durations do not map
+    to human time periods (e.g. weeks or hours).
+
+    Parameters
+    ----------
+    p : int
+        cast an integer to the specified pointwidth
+    """
+
+    @classmethod
+    def from_timedelta(cls, delta):
+        """
+        Returns the closest pointwidth for the given timedelta without going over the
+        specified duration. Because pointwidths are in powers of 2, be sure to check
+        that the returned real duration is sufficient.
+        """
+        return cls.from_nanoseconds(delta.total_seconds()*1e9)
+
+    @classmethod
+    def from_nanoseconds(cls, nsec):
+        """
+        Returns the closest pointwidth for the given number of nanoseconds without going
+        over the specified duration. Because pointwidths are in powers of 2, be sure to
+        check that the returned real duration is sufficient.
+        """
+        for pos in range(62):
+            nsec = nsec >> 1
+            if nsec == 0:
+                break
+        return cls(pos)
+
+    def __init__(self, p):
+        self._pointwidth = int(p)
+
+    @property
+    def nanoseconds(self):
+        return 2**self._pointwidth
+
+    @property
+    def microseconds(self):
+        return self.nanoseconds / 1e3
+
+    @property
+    def milliseconds(self):
+        return self.nanoseconds / 1e6
+
+    @property
+    def seconds(self):
+        return self.nanoseconds / 1e9
+
+    @property
+    def minutes(self):
+        return self.nanoseconds / 6e10
+
+    @property
+    def hours(self):
+        return self.nanoseconds / 3.6e12
+
+    @property
+    def days(self):
+        return self.nanoseconds / 8.64e13
+
+    @property
+    def weeks(self):
+        return self.nanoseconds / 6.048e14
+
+    @property
+    def months(self):
+        return self.nanoseconds / 2.628e15
+
+    @property
+    def years(self):
+        return self.nanoseconds / 3.154e16
+
+    def decr(self):
+        return pointwidth(self-1)
+
+    def incr(self):
+        return pointwidth(self+1)
+
+    def __int__(self):
+        return self._pointwidth
+
+    def __eq__(self, other):
+        return int(self) == int(other)
+
+    def __lt__(self, other):
+        return int(self) < int(other)
+
+    def __le__(self, other):
+        return int(self) <= int(other)
+
+    def __gt__(self, other):
+        return int(self) > int(other)
+
+    def __ge__(self, other):
+        return int(self) >= int(other)
+
+    def __add__(self, other):
+        return pointwidth(int(self)+int(other))
+
+    def __sub__(self, other):
+        return pointwidth(int(self)-int(other))
+
+    def __str__(self):
+        """
+        Returns the pointwidth to the closest unit
+        """
+        if self >= 55:
+            return "{:0.2f} years".format(self.years)
+
+        if self >= 52:
+            return "{:0.2f} months".format(self.months)
+
+        if self >= 50:
+            return "{:0.2f} weeks".format(self.weeks)
+
+        if self >= 47:
+            return "{:0.2f} days".format(self.days)
+
+        if self >= 42:
+            return "{:0.2f} hours".format(self.hours)
+
+        if self >= 36:
+            return "{:0.2f} minutes".format(self.minutes)
+
+        if self >= 30:
+            return "{:0.2f} seconds".format(self.seconds)
+
+        if self >= 20:
+            return "{:0.2f} milliseconds".format(self.milliseconds)
+
+        if self >= 10:
+            return "{:0.2f} microseconds".format(self.microseconds)
+
+        return "{:0.0f} nanoseconds".format(self.nanoseconds)
+
+    def __repr__(self):
+        return "<pointwidth {}>".format(int(self))

btrdb/version.py

@@ -18,9 +18,9 @@
 __version_info__ = {
     'major': 5,
     'minor': 5,
-    'micro': 0,
+    'micro': 1,
     'releaselevel': 'final',
-    'serial': 8,
+    'serial': 9,
 }
 
 ##########################################################################

setup.cfg

@@ -11,3 +11,4 @@ python_files = tests/*
 flakes-ignore =
     tests/* ALL
     btrdb/grpcinterface/btrdb_pb2.py UnusedImport
+    btrdb/__init__.py UnusedImport

setup.py

@@ -129,8 +129,8 @@ def get_description_type(path=PKG_DESCRIBE):
     "description": DESCRIPTION,
     "long_description": read(PKG_DESCRIBE),
     "long_description_content_type": get_description_type(PKG_DESCRIBE),
-    "classifiers": CLASSIFIERS,
-    "keywords": KEYWORDS,
+    "classifiers": list(CLASSIFIERS),
+    "keywords": list(KEYWORDS),
     "license": LICENSE,
     "author": AUTHOR,
     "author_email": EMAIL,

tests/btrdb/test_base.py

@@ -26,7 +26,7 @@
 ## Test Constants
 ##########################################################################
 
-EXPECTED_VERSION = "5.5"
+EXPECTED_VERSION = "5.5.1"
 
 
 ##########################################################################