Merge pull request #128 from dennwc/v3

Python client v3 (UASTv2)

Author: juanjux

.travis.yml

@@ -1,22 +1,26 @@
 language: python
 sudo: true
 dist: xenial
+env: 
+- BBLFSHD_VERSION=v2.9.1 BBLFSH_PYTHON_VERSION=v2.3.0
 services:
   - docker
 cache:
   directories:
     - $HOME/.cache/pip
 python:
-  - "3.5"
   - "3.6"
   - "3.7"
 install:
+  - docker run --privileged -d -p 9432:9432 --name bblfshd bblfsh/bblfshd:$BBLFSHD_VERSION
+  - docker exec bblfshd bblfshctl driver install bblfsh/python-driver:$BBLFSH_PYTHON_VERSION
   - wget https://github.com/bblfsh/client-python/releases/download/v2.2.1/protobuf-python_3.4.1-1_amd64.deb
   - sudo dpkg -i protobuf-python_3.4.1-1_amd64.deb
   - pip3 install --upgrade pip
   - pip3 install -r requirements.txt
   - python3 setup.py --getdeps --log
   - pip3 install . --upgrade
+  - cd bblfsh && python3 -m unittest discover
   - if [[ -z "$TRAVIS_TAG" ]]; then exit 0; fi
   - if [[ $TRAVIS_PYTHON_VERSION != '3.6' ]]; then exit 0; fi # disable double uploads to pypi
   - echo "[distutils]" > .pypirc
@@ -28,6 +32,5 @@ install:
   - HOME=. python setup.py sdist upload
 script:
   - python3 setup.py build_ext -i
-  - python3 -m unittest discover .
 notifications:
   email: false

MANIFEST.in

@@ -7,5 +7,5 @@ include Makefile
 include github.com/gogo/protobuf/gogoproto/gogo.proto
 include gopkg.in/bblfsh/sdk.v1/protocol/generated.proto
 include gopkg.in/bblfsh/sdk.v1/uast/generated.proto
-include bblfsh/memtracker.h
+include bblfsh/libuast/libuast.hpp
 prune bblfsh/libuast

README.md

@@ -18,16 +18,18 @@ pip install bblfsh
 ```bash
 git clone https://github.com/bblfsh/client-python.git
 cd client-python
+pip install -r requirements.txt
+python setup.py --getdeps
 python setup.py install
+# or: pip install .
 ```
 
 ### Dependencies
 
-You need to install `libxml2` and its header files. You also will need a `curl` cli tool to dowload `libuast`, and a `g++` for building [libtuast Python bindings](https://github.com/bblfsh/client-python/blob/0037d762563ab49b3daac8a7577f7103a5628fc6/setup.py#L17). 
+You also will need a `curl` cli tool to dowload `libuast`, and a `g++` for building [libtuast Python bindings](https://github.com/bblfsh/client-python/blob/0037d762563ab49b3daac8a7577f7103a5628fc6/setup.py#L17).
 The command for Debian and derived distributions would be:
 
 ```bash
-sudo apt install libxml2-dev
 sudo apt install curl
 sudo apt install build-essential
 ```
@@ -49,21 +51,49 @@ Please, read the [getting started](https://doc.bblf.sh/using-babelfish/getting-s
 import bblfsh
 
 client = bblfsh.BblfshClient("0.0.0.0:9432")
-uast = client.parse("/path/to/file.py").uast
-print(uast)
-# "filter' allows you to use XPath queries to filter on result nodes:
-print(bblfsh.filter(uast, "//Import[@roleImport and @roleDeclaration]//alias"))
-
-# filter\_[bool|string|number] must be used when using XPath functions returning
-# these types:
-print(bblfsh.filter_bool(uast, "boolean(//*[@strtOffset or @endOffset])"))
-print(bblfsh.filter_string(uast, "name(//*[1])"))
-print(bblfsh.filter_number(uast, "count(//*)"))
+ctx = client.parse("/path/to/file.py")
+print(ctx)
+# or to get the results in a dictionary:
+resdict = ctx.get_all()
 
-# You can also iterate on several tree iteration orders:
-it = bblfsh.iterator(uast, bblfsh.TreeOrder.PRE_ORDER)
+# "filter' allows you to use XPath queries to filter on result nodes:
+it = ctx.filter("//python:Call")
 for node in it:
-    print(node.internal_type)
+    print(node)
+    # or:
+    doSomething(node.get())
+
+# filter must be used when using XPath functions returning these types:
+# XPath queries can return different types (dicts, int, float, bool or str), 
+# calling get() with an item will return the right type, but if you must ensure
+# that you are getting the expected type (to avoid errors in the queries) there
+# are alterative typed versions:
+x = next(ctx.filter("boolean(//*[@strtOffset or @endOffset])").get_bool()
+y = next(ctx.filter("name(//*[1])")).get_str()
+z = next(ctx.filter("count(//*)").get_int() # or get_float()
+
+# You can also iterate using iteration orders different than the 
+# default preorder using the `iterate` method on `parse` result or node objects:
+
+# Directly over parse results
+it = client.parse("/path/to/file.py").iterate(bblfsh.TreeOrder.POST_ORDER)
+for i in it: ...
+
+# Over filter results (which by default are already iterators with PRE_ORDER):
+ctx = client.parse("file.py")
+newiter = ctx.filter("//python:Call").iterate(bblfsh.TreeOrder.LEVEL_ORDER)
+for i in newiter: ...
+
+# Over individual node objects to change the iteration order of
+# a specific subtree:
+ctx = client.parse("file.py")
+first_node = next(ctx)
+newiter = first_node.iterate(bblfsh.TreeOrder.POSITION_ORDER)
+for i in newiter: ...
+
+# You can also get the non semantic UAST or native AST:
+ctx = client.parse("file.py", mode=bblfsh.ModeDict["NATIVE"])
+# Possible values for ModeDict: DEFAULT_MODE, NATIVE, PREPROCESSED, ANNOTATED, SEMANTIC
 ```
 
 Please read the [Babelfish clients](https://doc.bblf.sh/using-babelfish/clients.html)

bblfsh/__init__.py

@@ -1,33 +1,26 @@
 from bblfsh.client import BblfshClient
-from bblfsh.pyuast import filter, filter_bool, filter_number, filter_string, iterator
+from bblfsh.pyuast import decode, iterator, uast
+from bblfsh.tree_order import TreeOrder
 from bblfsh.aliases import *
 
-class TreeOrder:
-    PRE_ORDER      = 0
-    POST_ORDER     = 1
-    LEVEL_ORDER    = 2
-    POSITION_ORDER = 3
-
-# "in" is a reserved keyword in Python thus can't be used as package name, so
-# we import by string
 
 class RoleSearchException(Exception):
     pass
 
 
-def role_id(role_name: str) -> int:
+def role_id(rname: str) -> int:
     try:
-        name = DESCRIPTOR.enum_types_by_name["Role"].values_by_name[role_name].number
+        name = DESCRIPTOR.enum_types_by_name["Role"].values_by_name[rname].number
     except KeyError:
-        raise RoleSearchException("Role with name '{}' not found".format(role_name))
+        raise RoleSearchException("Role with name '{}' not found".format(rname))
 
     return name
 
 
-def role_name(role_id: int) -> str:
+def role_name(rid: int) -> str:
     try:
-        id_ = DESCRIPTOR.enum_types_by_name["Role"].values_by_number[role_id].name
+        id_ = DESCRIPTOR.enum_types_by_name["Role"].values_by_number[rid].name
     except KeyError:
-        raise RoleSearchException("Role with ID '{}' not found".format(role_id))
+        raise RoleSearchException("Role with ID '{}' not found".format(rid))
 
     return id_

bblfsh/__main__.py

@@ -1,69 +1,62 @@
 import argparse
+import pprint
 import sys
 
-import bblfsh
-from bblfsh.pyuast import filter
-
 from bblfsh.client import BblfshClient
 from bblfsh.launcher import ensure_bblfsh_is_running
 
 
-def setup():
+def setup() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Query for a UAST to Babelfish and dump it to stdout."
     )
     parser.add_argument("-e", "--endpoint", default="0.0.0.0:9432",
-                        help="bblfsh gRPC endpoint.")
+                        help="bblfsh gRPC endpoint.", type=str)
     parser.add_argument("-f", "--file", required=True,
-                        help="File to parse.")
+                        help="File to parse.", type=str)
     parser.add_argument("-l", "--language", default=None,
-                        help="File's language. The default is to autodetect.")
+                        help="File's language. The default is to autodetect.", type=str)
     parser.add_argument("--disable-bblfsh-autorun", action="store_true",
                         help="Do not automatically launch Babelfish server "
                              "if it is not running.")
 
-    parser.add_argument("-q", "--query", default="", help="xpath query")
-    parser.add_argument("-m", "--mapn", default="", help="transform function of the results (n)")
-    parser.add_argument("-a", "--array", help='print results as an array', action='store_true')
+    parser.add_argument("-q", "--query", default="", help="xpath query", type=str)
+    parser.add_argument("-a", "--array", help='print results as a parseable Python array', action='store_true')
 
-    args = parser.parse_args()
-    return args
+    return parser.parse_args()
 
-def run_query(root: bblfsh.Node, query: str, mapn: str, as_array: bool) -> None:
-    result = list(filter(root, query))
 
-    if not result:
+def run_query(uast, query: str, array: bool) -> None:
+    result_iter = uast.filter(query)
+    if not result_iter:
         print("Nothing found")
 
-    else:
-        if mapn:
-            result = [eval(mapn) for n in result]
+    result_list = [x.load() for x in result_iter]
 
-        if as_array:
-            print("results[{}] = {}".format(len(result), result))
-        else:
-            print("Running xpath query: {}".format(query))
-            print("FOUND {} roots".format(len(result)))
+    if array:
+        pprint.pprint(result_list)
+    else:
+        print("%d Results:" % len(result_list))
+        for i, node in enumerate(result_list):
+            print("== {} ==================================".format(i+1))
+            print(node)
 
-            for i, node in enumerate(result):
-                print("== {} ==================================".format(i+1))
-                print(node)
 
-def main():
+def main() -> int:
     args = setup()
     if not args.disable_bblfsh_autorun:
         ensure_bblfsh_is_running()
 
     client = BblfshClient(args.endpoint)
-    response = client.parse(args.file, args.language)
-    root = response.uast
-    if len(response.errors):
-        sys.stderr.write("\n".join(response.errors) + "\n")
-    query = args.query
-    if query:
-        run_query(root, query, args.mapn, args.array)
+    ctx = client.parse(args.file, args.language)
+
+    if args.query:
+        run_query(ctx, args.query, array=args.array)
     else:
-        print(root)
+        pprint.pprint(ctx.load())
+
+    return 0
+
 
 if __name__ == "__main__":
     sys.exit(main())

bblfsh/aliases.py

@@ -1,27 +1,39 @@
-__all__ = ["DESCRIPTOR", "Node", "Position", "ParseResponse", "NativeParseResponse",
-           "ParseRequest", "NativeParseRequest", "VersionRequest", "ProtocolServiceStub"]
-
 import importlib
-
-from bblfsh.sdkversion import VERSION
+import google
 
 # "in" is a reserved keyword in Python thus can't be used as package name, so
 # we import by string
-uast_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.uast.generated_pb2" % VERSION)
-protocol_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.protocol.generated_pb2" % VERSION)
-protocol_grpc_module = importlib.import_module(
-    "bblfsh.gopkg.in.bblfsh.sdk.%s.protocol.generated_pb2_grpc" % VERSION)
+uast_v2_module = importlib.import_module(
+    "bblfsh.gopkg.in.bblfsh.sdk.v2.uast.generated_pb2")
+protocol_v2_module = importlib.import_module(
+    "bblfsh.gopkg.in.bblfsh.sdk.v2.protocol.generated_pb2")
+protocol_grpc_v2_module = importlib.import_module(
+    "bblfsh.gopkg.in.bblfsh.sdk.v2.protocol.generated_pb2_grpc")
+protocol_v1_module = importlib.import_module(
+    "bblfsh.gopkg.in.bblfsh.sdk.v1.protocol.generated_pb2")
+protocol_grpc_v1_module = importlib.import_module(
+    "bblfsh.gopkg.in.bblfsh.sdk.v1.protocol.generated_pb2_grpc")
+
+DESCRIPTOR = uast_v2_module.DESCRIPTOR
+ParseRequest = protocol_v2_module.ParseRequest
+ParseResponse = protocol_v2_module.ParseResponse
+ParseError = protocol_v2_module.ParseError
+Mode = protocol_v2_module.Mode
+ModeType = google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper
+
+
+class Modes:
+    pass
+
+# Current values: {'DEFAULT_MODE': 0, 'NATIVE': 1, 'PREPROCESSED': 2, 'ANNOTATED': 4, 'SEMANTIC': 8}
+for k, v in Mode.DESCRIPTOR.values_by_name.items():
+    setattr(Modes, k, v.number)
+
+DriverStub = protocol_grpc_v2_module.DriverStub
+DriverServicer = protocol_grpc_v2_module.DriverServicer
 
-DESCRIPTOR = uast_module.DESCRIPTOR
-Node = uast_module.Node
-Position = uast_module.Position
-ParseResponse = protocol_module.ParseResponse
-NativeParseResponse = protocol_module.NativeParseResponse
-ParseRequest = protocol_module.ParseRequest
-NativeParseRequest = protocol_module.NativeParseRequest
-VersionRequest = protocol_module.VersionRequest
-SupportedLanguagesRequest = protocol_module.SupportedLanguagesRequest
-SupportedLanguagesResponse = protocol_module.SupportedLanguagesResponse
-ProtocolServiceStub = protocol_grpc_module.ProtocolServiceStub
+VersionRequest = protocol_v1_module.VersionRequest
+VersionResponse = protocol_v1_module.VersionResponse
+SupportedLanguagesRequest = protocol_v1_module.SupportedLanguagesRequest
+SupportedLanguagesResponse = protocol_v1_module.SupportedLanguagesResponse
+ProtocolServiceStub = protocol_grpc_v1_module.ProtocolServiceStub