Add support for new api

Author: DhruvSrikanth

examples/cookbook.ipynb

@@ -326,6 +326,13 @@
     "for key, value in query_response.items(): print(f\"{key}: {value}\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## A/B Testing with Function Versions"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

pyproject.toml

@@ -10,7 +10,7 @@ authors = [
 ]
 description = "A client facing API for interacting with the WeCo AI function builder service."
 readme = "README.md"
-version = "0.1.5"
+version = "0.1.6"
 license = {text = "MIT"}
 requires-python = ">=3.8"
 dependencies = ["asyncio", "httpx[http2]", "pillow"]

tests/test_asynchronous.py

@@ -8,8 +8,9 @@
 @pytest.mark.asyncio
 async def test_abuild(text_evaluator, image_evaluator, text_and_image_evaluator):
     for evaluator in [text_evaluator, image_evaluator, text_and_image_evaluator]:
-        fn_name, fn_desc = await evaluator
+        fn_name, version_number, fn_desc = await evaluator
         assert isinstance(fn_name, str)
+        assert isinstance(version_number, int)
         assert isinstance(fn_desc, str)
 
 
@@ -23,34 +24,37 @@ async def assert_query_response(query_response):
 
 @pytest.fixture
 async def text_evaluator():
-    fn_name, fn_desc = await abuild(
-        task_description="Evaluate the sentiment of the given text. Provide a json object with 'sentiment' and 'explanation' keys."
+    fn_name, version_number, fn_desc = await abuild(
+        task_description="Evaluate the sentiment of the given text. Provide a json object with 'sentiment' and 'explanation' keys.",
+        multimodal=False
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 @pytest.mark.asyncio
 async def test_text_aquery(text_evaluator):
-    fn_name, _ = await text_evaluator
-    query_response = await aquery(fn_name=fn_name, text_input="I love this product!")
+    fn_name, version_number, _ = await text_evaluator
+    query_response = await aquery(fn_name=fn_name, version_number=version_number, text_input="I love this product!")
 
     await assert_query_response(query_response)
     assert set(query_response["output"].keys()) == {"sentiment", "explanation"}
 
 
 @pytest.fixture
 async def image_evaluator():
-    fn_name, fn_desc = await abuild(
-        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys."
+    fn_name, version_number, fn_desc = await abuild(
+        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys.",
+        multimodal=True
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 @pytest.mark.asyncio
 async def test_image_aquery(image_evaluator):
-    fn_name, _ = await image_evaluator
+    fn_name, version_number, _ = await image_evaluator
     query_response = await aquery(
         fn_name=fn_name,
+        version_number=version_number,
         images_input=[
             "https://www.integratedtreatmentservices.co.uk/wp-content/uploads/2013/12/Objects-of-Reference.jpg",
             "https://t4.ftcdn.net/jpg/05/70/90/23/360_F_570902339_kNj1reH40GFXakTy98EmfiZHci2xvUCS.jpg",
@@ -63,17 +67,19 @@ async def test_image_aquery(image_evaluator):
 
 @pytest.fixture
 async def text_and_image_evaluator():
-    fn_name, fn_desc = await abuild(
-        task_description="Evaluate, solve and arrive at a numerical answer for the image provided. Provide a json object with 'answer' and 'explanation' keys."
+    fn_name, version_number, fn_desc = await abuild(
+        task_description="Evaluate, solve and arrive at a numerical answer for the image provided. Provide a json object with 'answer' and 'explanation' keys.",
+        multimodal=True
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 @pytest.mark.asyncio
 async def test_text_and_image_aquery(text_and_image_evaluator):
-    fn_name, _ = await text_and_image_evaluator
+    fn_name, version_number, _ = await text_and_image_evaluator
     query_response = await aquery(
         fn_name=fn_name,
+        version_number=version_number,
         text_input="Find x and y.",
         images_input=[
             "https://i.ytimg.com/vi/cblHUeq3bkE/hq720.jpg?sqp=-oaymwEhCK4FEIIDSFryq4qpAxMIARUAAAAAGAElAADIQj0AgKJD&rs=AOn4CLAKn3piY91QRCBzRgnzAPf7MPrjDQ"

tests/test_batching.py

@@ -5,62 +5,71 @@
 
 # Internally, these functions use the WecoAI client
 # therefore, we can test both the client and functional forms here
-# We do NOT need to test for a list of function names here since interally
-# we cast the input to a list of function names to match the fn_input size
 @pytest.fixture
 def ml_task_evaluator():
-    fn_name, _ = build(
-        task_description="I want to evaluate the feasibility of a machine learning task. Give me a json object with three keys - 'feasibility', 'justification', and 'suggestions'."
+    fn_name, version_number, _ = build(
+        task_description="I want to evaluate the feasibility of a machine learning task. Give me a json object with three keys - 'feasibility', 'justification', and 'suggestions'.",
+        multimodal=False
     )
-    return fn_name
+    return fn_name, version_number
 
 
 @pytest.fixture
 def ml_task_inputs():
     return [
         {"text_input": "I want to train a model to predict house prices using the Boston Housing dataset hosted on Kaggle."},
-        {
-            "text_input": "I want to train a model to classify digits using the MNIST dataset hosted on Kaggle using a Google Colab notebook."
-        },
+        {"text_input": "I want to train a model to classify digits using the MNIST dataset hosted on Kaggle using a Google Colab notebook."},
     ]
 
 
 @pytest.fixture
 def image_evaluator():
-    fn_name, _ = build(
-        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys."
+    fn_name, version_number, _ = build(
+        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys.",
+        multimodal=True
     )
-    return fn_name
+    return fn_name, version_number
 
 
 @pytest.fixture
 def image_inputs():
     return [
-        {
-            "images_input": [
-                "https://www.integratedtreatmentservices.co.uk/wp-content/uploads/2013/12/Objects-of-Reference.jpg"
-            ]
-        },
+        {"images_input": ["https://www.integratedtreatmentservices.co.uk/wp-content/uploads/2013/12/Objects-of-Reference.jpg"]},
         {"images_input": ["https://t4.ftcdn.net/jpg/05/70/90/23/360_F_570902339_kNj1reH40GFXakTy98EmfiZHci2xvUCS.jpg"]},
     ]
 
 
-def test_batch_query(ml_task_evaluator, ml_task_inputs, image_evaluator, image_inputs):
-    fn_names = [ml_task_evaluator, ml_task_evaluator, image_evaluator, image_evaluator]
-    batch_inputs = ml_task_inputs + image_inputs
+def test_batch_query_text(ml_task_evaluator, ml_task_inputs):
+    fn_name, version_number = ml_task_evaluator
+    batch_inputs = ml_task_inputs
 
-    query_responses = batch_query(fn_names=fn_names, batch_inputs=batch_inputs)
+    query_responses = batch_query(fn_name=fn_name, version_number=version_number, batch_inputs=batch_inputs)
 
     assert len(query_responses) == len(batch_inputs)
 
-    for i, query_response in enumerate(query_responses):
+    for query_response in query_responses:
         assert isinstance(query_response["output"], dict)
         assert isinstance(query_response["in_tokens"], int)
         assert isinstance(query_response["out_tokens"], int)
         assert isinstance(query_response["latency_ms"], float)
 
         output = query_response["output"]
-        if i < len(ml_task_inputs):
-            assert set(output.keys()) == {"feasibility", "justification", "suggestions"}
-        else:
-            assert set(output.keys()) == {"description", "objects"}
+        assert set(output.keys()) == {"feasibility", "justification", "suggestions"}
+
+
+def test_batch_query_image(image_evaluator, image_inputs):
+    fn_name, version_number = image_evaluator
+    batch_inputs = image_inputs
+
+    query_responses = batch_query(fn_name=fn_name, version_number=version_number, batch_inputs=batch_inputs)
+
+    assert len(query_responses) == len(batch_inputs)
+
+    for query_response in query_responses:
+        assert isinstance(query_response["output"], dict)
+        assert isinstance(query_response["in_tokens"], int)
+        assert isinstance(query_response["out_tokens"], int)
+        assert isinstance(query_response["latency_ms"], float)
+
+        output = query_response["output"]
+        assert set(output.keys()) == {"description", "objects"}

tests/test_synchronous.py

@@ -7,8 +7,9 @@
 # therefore, we can test both the client and functional forms here
 def test_build(text_evaluator, image_evaluator, text_and_image_evaluator):
     for evaluator in [text_evaluator, image_evaluator, text_and_image_evaluator]:
-        fn_name, fn_desc = evaluator
+        fn_name, version_number, fn_desc = evaluator
         assert isinstance(fn_name, str)
+        assert isinstance(version_number, int)
         assert isinstance(fn_desc, str)
 
 
@@ -22,32 +23,35 @@ def assert_query_response(query_response):
 
 @pytest.fixture
 def text_evaluator():
-    fn_name, fn_desc = build(
-        task_description="Evaluate the sentiment of the given text. Provide a json object with 'sentiment' and 'explanation' keys."
+    fn_name, version_number, fn_desc = build(
+        task_description="Evaluate the sentiment of the given text. Provide a json object with 'sentiment' and 'explanation' keys.",
+        multimodal=False
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 def test_text_query(text_evaluator):
-    fn_name, _ = text_evaluator
-    query_response = query(fn_name=fn_name, text_input="I love this product!")
+    fn_name, version_number, _ = text_evaluator
+    query_response = query(fn_name=fn_name, version_number=version_number, text_input="I love this product!")
 
     assert_query_response(query_response)
     assert set(query_response["output"].keys()) == {"sentiment", "explanation"}
 
 
 @pytest.fixture
 def image_evaluator():
-    fn_name, fn_desc = build(
-        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys."
+    fn_name, version_number, fn_desc = build(
+        task_description="Describe the contents of the given images. Provide a json object with 'description' and 'objects' keys.",
+        multimodal=True
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 def test_image_query(image_evaluator):
-    fn_name, _ = image_evaluator
+    fn_name, version_number, _ = image_evaluator
     query_response = query(
         fn_name=fn_name,
+        version_number=version_number,
         images_input=[
             "https://www.integratedtreatmentservices.co.uk/wp-content/uploads/2013/12/Objects-of-Reference.jpg",
             "https://t4.ftcdn.net/jpg/05/70/90/23/360_F_570902339_kNj1reH40GFXakTy98EmfiZHci2xvUCS.jpg",
@@ -60,16 +64,18 @@ def test_image_query(image_evaluator):
 
 @pytest.fixture
 def text_and_image_evaluator():
-    fn_name, fn_desc = build(
-        task_description="Evaluate, solve and arrive at a numerical answer for the image provided. Perform any additional things if instructed. Provide a json object with 'answer' and 'explanation' keys."
+    fn_name, version_number, fn_desc = build(
+        task_description="Evaluate, solve and arrive at a numerical answer for the image provided. Perform any additional things if instructed. Provide a json object with 'answer' and 'explanation' keys.",
+        multimodal=True
     )
-    return fn_name, fn_desc
+    return fn_name, version_number, fn_desc
 
 
 def test_text_and_image_query(text_and_image_evaluator):
-    fn_name, _ = text_and_image_evaluator
+    fn_name, version_number, _ = text_and_image_evaluator
     query_response = query(
         fn_name=fn_name,
+        version_number=version_number,
         text_input="Find x and y.",
         images_input=[
             "https://i.ytimg.com/vi/cblHUeq3bkE/hq720.jpg?sqp=-oaymwEhCK4FEIIDSFryq4qpAxMIARUAAAAAGAElAADIQj0AgKJD&rs=AOn4CLAKn3piY91QRCBzRgnzAPf7MPrjDQ"