Add support for wheels when deploying pipelines (#249)

* add support for wheels

* linter

Author: arulajmani

databricks_cli/pipelines/api.py

@@ -38,6 +38,7 @@
 
 BUFFER_SIZE = 1024 * 64
 base_pipelines_dir = 'dbfs:/pipelines/code'
+supported_lib_types = {'jar', 'whl'}
 
 
 class PipelinesApi(object):
@@ -73,9 +74,9 @@ def _identify_local_libraries(lib_objects):
         local_lib_objects, external_lib_objects = [], []
         for lib_object in lib_objects:
             parsed_uri = urllib.parse.urlparse(lib_object.path)
-            if lib_object.lib_type == 'jar' and parsed_uri.scheme == '':
+            if lib_object.lib_type in supported_lib_types and parsed_uri.scheme == '':
                 local_lib_objects.append(lib_object)
-            elif lib_object.lib_type == 'jar' and parsed_uri.scheme.lower() == 'file':
+            elif lib_object.lib_type in supported_lib_types and parsed_uri.scheme.lower() == 'file':
                 # exactly 1 or 3
                 if parsed_uri.path.startswith('//') or parsed_uri.netloc != '':
                     raise RuntimeError('invalid file uri scheme, '

tests/pipelines/test_api.py

@@ -151,31 +151,31 @@ def test_partition_local_remote(pipelines_api):
         LibraryObject('jar', 'FILE:/all/caps.ext'),
         LibraryObject('jar', 'FiLe:/weird/case.ext'),
         LibraryObject('jar', 'file.ext'),
+        LibraryObject('whl', 'rel/path.ext'),
         # shouldn't be uploaded
         LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
         LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
         LibraryObject('jar', 'scheme:file.ext'),
         LibraryObject('jar', 'scheme:/abs/path.ext'),
         LibraryObject('jar', 'scheme://abs/path.ext'),
-        LibraryObject('egg', 'file:/abs/path.ext'),
-        LibraryObject('whl', 'rel/path.ext')
+        LibraryObject('egg', 'file:/abs/path.ext')
     ]
     expected_llo = [
         LibraryObject('jar', '/absolute/path/abc.ext'),
         LibraryObject('jar', 'relative/path.ext'),
         LibraryObject('jar', '/file/scheme/abs/path.ext'),
         LibraryObject('jar', '/all/caps.ext'),
         LibraryObject('jar', '/weird/case.ext'),
-        LibraryObject('jar', 'file.ext')
+        LibraryObject('jar', 'file.ext'),
+        LibraryObject('whl', 'rel/path.ext')
     ]
     expected_external = [
         LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'),
         LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'),
         LibraryObject('jar', 'scheme:file.ext'),
         LibraryObject('jar', 'scheme:/abs/path.ext'),
         LibraryObject('jar', 'scheme://abs/path.ext'),
-        LibraryObject('egg', 'file:/abs/path.ext'),
-        LibraryObject('whl', 'rel/path.ext')
+        LibraryObject('egg', 'file:/abs/path.ext')
     ]
     llo, external = pipelines_api._identify_local_libraries(libraries)
     assert llo == expected_llo
@@ -207,4 +207,4 @@ def test_library_object_serialization_deserialization():
     assert llo == library_objects
 
     libs = LibraryObject.to_json(library_objects)
-    assert libs == libraries
+    assert libs == libraries