Skip to content

Commit b4198cc

Browse files
max-deygin-traceloopmax-deygin-traceloop
committed
Update test assertions
1 parent 9b05f3a commit b4198cc

1 file changed

Lines changed: 46 additions & 75 deletions

File tree

‎packages/opentelemetry-instrumentation-openai/tests/traces/test_assistant.py‎

Lines changed: 46 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,8 @@ def test_new_assistant(
7979
)
8080
assert input_messages[2]["role"] == "user"
8181
assert input_messages[2]["parts"][0]["content"] == user_message
82-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 145
83-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 155
82+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
83+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 89
8484
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
8585

8686
output_messages = get_output_messages(open_ai_span)
@@ -150,8 +150,8 @@ def test_new_assistant_with_events_with_content(
150150
== "gpt-4o-mini"
151151
)
152152

153-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 145
154-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 155
153+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
154+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
155155
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
156156

157157
logs = log_exporter.get_finished_logs()
@@ -230,8 +230,8 @@ def test_new_assistant_with_events_with_no_content(
230230
open_ai_span.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL]
231231
== "gpt-4o-mini"
232232
)
233-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 145
234-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 155
233+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
234+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
235235
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
236236

237237
logs = log_exporter.get_finished_logs()
@@ -301,8 +301,8 @@ def test_new_assistant_with_polling(
301301
)
302302
assert input_messages[2]["role"] == "user"
303303
assert input_messages[2]["parts"][0]["content"] == user_message
304-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 374
305-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 86
304+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
305+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
306306
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
307307

308308
output_messages = get_output_messages(open_ai_span)
@@ -357,8 +357,8 @@ def test_new_assistant_with_polling_with_events_with_content(
357357
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat"
358358
assert open_ai_span.attributes["gen_ai.request.model"] == "gpt-4o-mini"
359359
assert open_ai_span.attributes["gen_ai.response.model"] == "gpt-4o-mini"
360-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 374
361-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 86
360+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 376
361+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 83
362362
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
363363

364364
logs = log_exporter.get_finished_logs()
@@ -431,8 +431,8 @@ def test_new_assistant_with_polling_with_events_with_no_content(
431431
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == "chat"
432432
assert open_ai_span.attributes["gen_ai.request.model"] == "gpt-4o-mini"
433433
assert open_ai_span.attributes["gen_ai.response.model"] == "gpt-4o-mini"
434-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 374
435-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 86
434+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
435+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 89
436436
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
437437

438438
logs = log_exporter.get_finished_logs()
@@ -468,7 +468,7 @@ def test_existing_assistant(
468468

469469
run = openai_client.beta.threads.runs.create(
470470
thread_id=thread.id,
471-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
471+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
472472
instructions="Please address the user as Jane Doe. The user has a premium account.",
473473
)
474474

@@ -512,8 +512,8 @@ def test_existing_assistant(
512512
)
513513
assert input_messages[2]["role"] == "user"
514514
assert input_messages[2]["parts"][0]["content"] == user_message
515-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 639
516-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 170
515+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
516+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
517517
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
518518

519519
output_messages = get_output_messages(open_ai_span)
@@ -547,7 +547,7 @@ def test_existing_assistant_with_events_with_content(
547547

548548
run = openai_client.beta.threads.runs.create(
549549
thread_id=thread.id,
550-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
550+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
551551
instructions="Please address the user as Jane Doe. The user has a premium account.",
552552
)
553553

@@ -579,12 +579,12 @@ def test_existing_assistant_with_events_with_content(
579579
== "gpt-4o-mini"
580580
)
581581

582-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 639
583-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 170
582+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
583+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
584584
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
585585

586586
logs = log_exporter.get_finished_logs()
587-
assert len(logs) == 5
587+
assert len(logs) == 4
588588

589589
# Validate run system message Event
590590
assert_message_in_logs(
@@ -612,21 +612,13 @@ def test_existing_assistant_with_events_with_content(
612612
{"content": user_message},
613613
)
614614

615-
# Validate the first ai response
615+
# Validate the ai response
616616
choice_event = {
617617
"index": 0,
618618
"finish_reason": "unknown",
619-
"message": {"content": messages.data[-2].content[0].text.value},
620-
}
621-
assert_message_in_logs(logs[3], "gen_ai.choice", choice_event)
622-
623-
# Validate the second ai response
624-
choice_event = {
625-
"index": 1,
626-
"finish_reason": "unknown",
627619
"message": {"content": messages.data[-1].content[0].text.value},
628620
}
629-
assert_message_in_logs(logs[4], "gen_ai.choice", choice_event)
621+
assert_message_in_logs(logs[3], "gen_ai.choice", choice_event)
630622

631623

632624
@pytest.mark.vcr
@@ -644,7 +636,7 @@ def test_existing_assistant_with_events_with_no_content(
644636

645637
run = openai_client.beta.threads.runs.create(
646638
thread_id=thread.id,
647-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
639+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
648640
instructions="Please address the user as Jane Doe. The user has a premium account.",
649641
)
650642

@@ -674,12 +666,12 @@ def test_existing_assistant_with_events_with_no_content(
674666
== "gpt-4o-mini"
675667
)
676668

677-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 639
678-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 170
669+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
670+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 89
679671
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
680672

681673
logs = log_exporter.get_finished_logs()
682-
assert len(logs) == 5
674+
assert len(logs) == 4
683675

684676
# Validate run system message Event
685677
assert_message_in_logs(logs[0], "gen_ai.system.message", {})
@@ -691,14 +683,10 @@ def test_existing_assistant_with_events_with_no_content(
691683
user_message_log = logs[2]
692684
assert_message_in_logs(user_message_log, "gen_ai.user.message", {})
693685

694-
# Validate the first ai response
686+
# Validate the ai response
695687
choice_event = {"index": 0, "finish_reason": "unknown", "message": {}}
696688
assert_message_in_logs(logs[3], "gen_ai.choice", choice_event)
697689

698-
# Validate the second ai response
699-
choice_event = {"index": 1, "finish_reason": "unknown", "message": {}}
700-
assert_message_in_logs(logs[4], "gen_ai.choice", choice_event)
701-
702690

703691
@pytest.mark.vcr
704692
def test_streaming_new_assistant(
@@ -758,8 +746,8 @@ def on_text_delta(self, delta, snapshot):
758746
" The user has a premium account."
759747
)
760748

761-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 790
762-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 225
749+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
750+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
763751
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
764752

765753
output_messages = get_output_messages(open_ai_span)
@@ -821,12 +809,12 @@ def on_text_delta(self, delta, snapshot):
821809
== "gpt-4o-mini"
822810
)
823811

824-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 790
825-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 225
812+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
813+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 89
826814
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
827815

828816
logs = log_exporter.get_finished_logs()
829-
assert len(logs) == 4
817+
assert len(logs) == 3
830818

831819
# Validate run system message Event
832820
assert_message_in_logs(
@@ -861,21 +849,6 @@ def on_text_delta(self, delta, snapshot):
861849
}
862850
assert_message_in_logs(logs[2], "gen_ai.choice", choice_event)
863851

864-
# Validate the second ai response
865-
choice_event = {
866-
"index": 1,
867-
"finish_reason": "unknown",
868-
"message": {
869-
"content": [
870-
{
871-
"text": {"annotations": [], "value": assistant_messages[1]},
872-
"type": "text",
873-
}
874-
],
875-
},
876-
}
877-
assert_message_in_logs(logs[3], "gen_ai.choice", choice_event)
878-
879852

880853
@pytest.mark.vcr
881854
def test_streaming_new_assistant_with_events_with_no_content(
@@ -924,12 +897,12 @@ def on_text_delta(self, delta, snapshot):
924897
== "gpt-4o-mini"
925898
)
926899

927-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 790
928-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 225
900+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 382
901+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 90
929902
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
930903

931904
logs = log_exporter.get_finished_logs()
932-
assert len(logs) == 4
905+
assert len(logs) == 3
933906

934907
# Validate run system message Event
935908
assert_message_in_logs(logs[0], "gen_ai.system.message", {})
@@ -941,10 +914,6 @@ def on_text_delta(self, delta, snapshot):
941914
choice_event = {"index": 0, "finish_reason": "unknown", "message": {}}
942915
assert_message_in_logs(logs[2], "gen_ai.choice", choice_event)
943916

944-
# Validate the second ai response
945-
choice_event = {"index": 1, "finish_reason": "unknown", "message": {}}
946-
assert_message_in_logs(logs[3], "gen_ai.choice", choice_event)
947-
948917

949918
@pytest.mark.vcr
950919
def test_streaming_existing_assistant(
@@ -971,7 +940,7 @@ def on_text_delta(self, delta, snapshot):
971940

972941
with openai_client.beta.threads.runs.create_and_stream(
973942
thread_id=thread.id,
974-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
943+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
975944
instructions="Please address the user as Jane Doe. The user has a premium account.",
976945
event_handler=EventHandler(),
977946
) as stream:
@@ -1003,8 +972,8 @@ def on_text_delta(self, delta, snapshot):
1003972
"Please address the user as Jane Doe."
1004973
" The user has a premium account."
1005974
)
1006-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 364
1007-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
975+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
976+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 89
1008977
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
1009978

1010979
output_messages = get_output_messages(open_ai_span)
@@ -1044,7 +1013,7 @@ def on_text_delta(self, delta, snapshot):
10441013

10451014
with openai_client.beta.threads.runs.create_and_stream(
10461015
thread_id=thread.id,
1047-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
1016+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
10481017
instructions="Please address the user as Jane Doe. The user has a premium account.",
10491018
event_handler=EventHandler(),
10501019
) as stream:
@@ -1066,8 +1035,8 @@ def on_text_delta(self, delta, snapshot):
10661035
== "gpt-4o-mini"
10671036
)
10681037

1069-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 364
1070-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
1038+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 380
1039+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 90
10711040
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
10721041

10731042
logs = log_exporter.get_finished_logs()
@@ -1078,7 +1047,8 @@ def on_text_delta(self, delta, snapshot):
10781047
logs[0],
10791048
"gen_ai.system.message",
10801049
{
1081-
"content": "You are a personal math tutor. Write and run code to answer math questions.",
1050+
"content": "You are a personal math tutor."
1051+
" Write and run code to answer math questions."
10821052
},
10831053
)
10841054

@@ -1087,7 +1057,8 @@ def on_text_delta(self, delta, snapshot):
10871057
logs[1],
10881058
"gen_ai.system.message",
10891059
{
1090-
"content": "Please address the user as Jane Doe. The user has a premium account.",
1060+
"content": "Please address the user as Jane Doe."
1061+
" The user has a premium account."
10911062
},
10921063
)
10931064

@@ -1132,7 +1103,7 @@ def on_text_delta(self, delta, snapshot):
11321103

11331104
with openai_client.beta.threads.runs.create_and_stream(
11341105
thread_id=thread.id,
1135-
assistant_id="asst_rr3RGZE5iqoMCxqFOpb7AZmr",
1106+
assistant_id="asst_D2BqLQ9rNgOjd5A3NpS9m9s4",
11361107
instructions="Please address the user as Jane Doe. The user has a premium account.",
11371108
event_handler=EventHandler(),
11381109
) as stream:
@@ -1154,7 +1125,7 @@ def on_text_delta(self, delta, snapshot):
11541125
== "gpt-4o-mini"
11551126
)
11561127

1157-
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 364
1128+
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 381
11581129
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 88
11591130
assert open_ai_span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai"
11601131

0 commit comments

Comments
 (0)