HN Listener UI fixes (#10)

yanomaly · samjulien · commit b1940180610c · 2024-12-23T09:37:41.000-08:00
- Removed backend driven UI usage and replaced it by Repeater component
- Removed messages visibility flags
- Fixed validation of scrapped data
diff --git a/hacker-news-social-listener/.wf/components-page-0-c0f99a9e-5004-4e75-a6c6-36f17490b134.jsonl b/hacker-news-social-listener/.wf/components-page-0-c0f99a9e-5004-4e75-a6c6-36f17490b134.jsonl
@@ -3,7 +3,7 @@
 {"id": "m7luxumzscv65i09", "type": "tabs", "content": {}, "handlers": {}, "isCodeManaged": false, "parentId": "c0f99a9e-5004-4e75-a6c6-36f17490b134", "position": 1, "visible": {"binding": "", "expression": true, "reversed": false}}
 {"id": "lon4vs20gd3myh7e", "type": "tab", "content": {"name": "Setup"}, "handlers": {}, "isCodeManaged": false, "parentId": "m7luxumzscv65i09", "position": 0, "visible": {"binding": "", "expression": true, "reversed": false}}
 {"id": "xlc1wtcwu1g2i07p", "type": "heading", "content": {"text": "Navigate and analyze Hacker News posts and comments using Writer graph-based RAG and Palmyra X 004."}, "handlers": {}, "isCodeManaged": false, "parentId": "lon4vs20gd3myh7e", "position": 0}
-{"id": "62r9auy895datfnp", "type": "message", "content": {"message": "@{message_setup}"}, "handlers": {}, "isCodeManaged": false, "parentId": "lon4vs20gd3myh7e", "position": 1, "visible": {"binding": "message_setup_vis", "expression": "custom", "reversed": false}}
+{"id": "62r9auy895datfnp", "type": "message", "content": {"message": "@{message_setup}"}, "handlers": {}, "isCodeManaged": false, "parentId": "lon4vs20gd3myh7e", "position": 1, "visible": {"binding": "message_setup_vis", "expression": true, "reversed": false}}
 {"id": "mmh30t1tiyv5mi9s", "type": "section", "content": {"containerBackgroundColor": "#BFCBFF", "isCollapsible": "yes", "title": "\ud83d\udd0e Pull latest N posts"}, "handlers": {}, "isCodeManaged": false, "parentId": "lon4vs20gd3myh7e", "position": 2}
 {"id": "hpeff7ha6v27zsk8", "type": "text", "content": {"text": "**Pull up to 500 Hacker News stories, with or without comments.** Data from the Hacker News API is sent to to Writer\u2019s graph-based RAG system, known as a Knowledge Graph. This is accessed dynamically by an LLM through the Writer RAG tool.", "useMarkdown": "yes"}, "handlers": {}, "isCodeManaged": false, "parentId": "mmh30t1tiyv5mi9s", "position": 0}
 {"id": "up5ofq0drv233umy", "type": "section", "content": {"containerBackgroundColor": "#FFD8CD", "isCollapsible": "yes", "title": "\ud83d\ude4b\u200d\u2640\ufe0f Ask questions"}, "handlers": {}, "isCodeManaged": false, "parentId": "lon4vs20gd3myh7e", "position": 3}
@@ -25,14 +25,19 @@
 {"id": "crm2bdbrjclid4k4", "type": "chatbot", "content": {"conversation": "@{conversation}", "useMarkdown": "yes"}, "handlers": {"wf-chatbot-message": "message_handler"}, "isCodeManaged": false, "parentId": "rosw32keaejygiir", "position": 0}
 {"id": "814lb9dktd2e1ye3", "type": "horizontalstack", "content": {"contentHAlign": "start"}, "handlers": {}, "isCodeManaged": false, "parentId": "rosw32keaejygiir", "position": 1}
 {"id": "t171dwz5muor2n9x", "type": "text", "content": {"alignment": "center", "text": "@{contributing_sources_button_text}", "useMarkdown": "yes"}, "handlers": {"wf-click": "contributing_sources_change_vis"}, "isCodeManaged": false, "parentId": "814lb9dktd2e1ye3", "position": 0}
-{"id": "contributed_sources", "type": "column", "content": {"cssClasses": "files-list", "title": "Contributing sources", "width": "7"}, "handlers": {}, "isCodeManaged": false, "parentId": "rpi88dvxmlxr0qd9", "position": 1, "visible": {"binding": "contributing_sources_vis", "expression": "custom", "reversed": false}}
+{"id": "lcjq00s0z6a4cv6f", "type": "column", "content": {"title": "Contributing sources", "width": "7"}, "handlers": {}, "isCodeManaged": false, "parentId": "rpi88dvxmlxr0qd9", "position": 1, "visible": {"binding": "contributing_sources_vis", "expression": "custom", "reversed": false}}
+{"id": "pn1m5c8qgt9s24a8", "type": "columns", "content": {}, "handlers": {}, "isCodeManaged": false, "parentId": "lcjq00s0z6a4cv6f", "position": 0}
+{"id": "contributed_sources", "type": "column", "content": {"cssClasses": "files-list", "title": "", "width": "7"}, "handlers": {}, "isCodeManaged": false, "parentId": "pn1m5c8qgt9s24a8", "position": 0, "visible": {"binding": "contributing_sources_vis", "expression": "custom", "reversed": false}}
+{"id": "vncphxzffznqjhvv", "type": "repeater", "content": {"keyVariable": "index", "repeaterObject": "@{contributing_sources}", "valueVariable": "file"}, "handlers": {}, "isCodeManaged": false, "parentId": "contributed_sources", "position": 0}
+{"id": "e1s5oy7xr2v7jlas", "type": "section", "content": {"cssClasses": "@{file.file_css}", "title": "@{file.name}"}, "handlers": {}, "isCodeManaged": false, "parentId": "vncphxzffznqjhvv", "position": 0}
+{"id": "01fzo8tvm1dop7us", "type": "text", "content": {"cssClasses": "@{file.content_css}", "text": "@{file.content}", "useMarkdown": "yes"}, "handlers": {}, "isCodeManaged": false, "parentId": "e1s5oy7xr2v7jlas", "position": 0}
 {"id": "erkqharzgysk5s3n", "type": "tab", "content": {"name": "Generate trend report"}, "handlers": {}, "isCodeManaged": false, "parentId": "m7luxumzscv65i09", "position": 3, "visible": {"binding": "", "expression": true, "reversed": false}}
-{"id": "vzbrrx0dlp3skcn2", "type": "message", "content": {"message": "@{message_report}"}, "handlers": {}, "isCodeManaged": false, "parentId": "erkqharzgysk5s3n", "position": 0, "visible": {"binding": "message_report_vis", "expression": "custom", "reversed": false}}
+{"id": "vzbrrx0dlp3skcn2", "type": "message", "content": {"message": "@{message_report}"}, "handlers": {}, "isCodeManaged": false, "parentId": "erkqharzgysk5s3n", "position": 0, "visible": {"binding": "message_report_vis", "expression": true, "reversed": false}}
 {"id": "osvuzxnivfs3qrut", "type": "text", "content": {"text": "@{prepared_report}", "useMarkdown": "yes"}, "handlers": {}, "isCodeManaged": false, "parentId": "erkqharzgysk5s3n", "position": 1}
 {"id": "w2hg99rv80b12tpt", "type": "button", "content": {"text": "Generate report"}, "handlers": {"wf-click": "run_report"}, "isCodeManaged": false, "parentId": "erkqharzgysk5s3n", "position": 2, "visible": {"binding": "", "expression": true, "reversed": false}}
-{"id": "so0q29s1ql8bsdj5", "type": "horizontalstack", "content": {}, "handlers": {}, "parentId": "c0f99a9e-5004-4e75-a6c6-36f17490b134", "position": 2}
-{"id": "n40gsxjweizjl009", "type": "link", "content": {"rel": "noopener noreferrer", "target": "_blank", "text": "Made with Writer Framework.", "url": "https://github.com/writer/writer-framework"}, "handlers": {}, "parentId": "so0q29s1ql8bsdj5", "position": 0}
-{"id": "s1sfmjca2cc9q0ho", "type": "text", "content": {"text": "|"}, "handlers": {}, "parentId": "so0q29s1ql8bsdj5", "position": 1}
+{"id": "so0q29s1ql8bsdj5", "type": "horizontalstack", "content": {}, "handlers": {}, "isCodeManaged": false, "parentId": "c0f99a9e-5004-4e75-a6c6-36f17490b134", "position": 2}
+{"id": "n40gsxjweizjl009", "type": "link", "content": {"rel": "noopener noreferrer", "target": "_blank", "text": "Made with Writer Framework.", "url": "https://github.com/writer/writer-framework"}, "handlers": {}, "isCodeManaged": false, "parentId": "so0q29s1ql8bsdj5", "position": 0}
+{"id": "s1sfmjca2cc9q0ho", "type": "text", "content": {"text": "|"}, "handlers": {}, "isCodeManaged": false, "parentId": "so0q29s1ql8bsdj5", "position": 1}
 {"id": "7a41wj8am7nk74we", "type": "link", "content": {"rel": "noopener noreferrer", "target": "_blank", "text": "Build your own RAG app.", "url": "https://writer.com/engineering/rag-tool/"}, "handlers": {}, "isCodeManaged": false, "parentId": "so0q29s1ql8bsdj5", "position": 2}
-{"id": "bxi8e7e2yexuifxo", "type": "text", "content": {"text": "|"}, "handlers": {}, "parentId": "so0q29s1ql8bsdj5", "position": 3}
-{"id": "0pjs3f2feqnik6oh", "type": "link", "content": {"rel": "noopener noreferrer", "target": "_blank", "text": "Get the code.", "url": "https://github.com/writer/framework-tutorials/tree/main/hacker-news-social-listener"}, "handlers": {}, "parentId": "so0q29s1ql8bsdj5", "position": 4}
+{"id": "bxi8e7e2yexuifxo", "type": "text", "content": {"text": "|"}, "handlers": {}, "isCodeManaged": false, "parentId": "so0q29s1ql8bsdj5", "position": 3}
+{"id": "0pjs3f2feqnik6oh", "type": "link", "content": {"rel": "noopener noreferrer", "target": "_blank", "text": "Get the code.", "url": "https://github.com/writer/framework-tutorials/tree/main/hacker-news-social-listener"}, "handlers": {}, "isCodeManaged": false, "parentId": "so0q29s1ql8bsdj5", "position": 4}
diff --git a/hacker-news-social-listener/main.py b/hacker-news-social-listener/main.py
@@ -33,28 +33,45 @@
 def main(state: WriterState) -> None:
     _delete_files_from_graph(GRAPH_ID)
     state["message_setup"] = "%Scraping data"
-    state["message_setup_vis"] = True
 
     posts, comments = _scrape_hackernews(state)
     state["message_setup"] = "%Data was scraped"
-    state["posts"] = posts[["title", "created_utc", "score", "num_comments", "url"]] if posts is not None else pd.DataFrame()
+    posts_columns = ["title", "created_utc", "score", "num_comments", "url"]
+    state["posts"] = (
+        posts[posts_columns]
+        if _verify_df(posts, posts_columns)
+        else pd.DataFrame(data={"Info": ["Invalid data was scrapped"]})
+    )
     if state["allow_comments"]:
-        state["comments"] = comments[["body", "author", "created_utc"]] if comments is not None else pd.DataFrame()
+        comments_columns = ["body", "author", "created_utc"]
+        state["comments"] = (
+            comments[comments_columns]
+            if _verify_df(comments, comments_columns)
+            else pd.DataFrame(data={"Info": ["Invalid data was scrapped"]})
+        )
     state["message_setup"] = "%Scraped data, now saving to csv"
 
     _save_results_to_csv(state)
     state["message_setup"] = "%Saved data, now uploading to KG"
 
     files_path = "static/hackernews_posts.csv"
-    _upload_file_and_add_to_graph(files_path, GRAPH_ID)
+    _upload_file_and_add_to_graph(state, files_path, GRAPH_ID)
     state["message_setup"] = "%Uploaded file and added to graph"
 
     if state["allow_comments"]:
         file_path = "static/hackernews_comments.csv"
-        _upload_file_and_add_to_graph(file_path, GRAPH_ID)
+        _upload_file_and_add_to_graph(state, file_path, GRAPH_ID)
+
+    state["message_setup"] = ""
 
-    state["message_setup"] = "+Scrapping is completed!"
-    state["message_setup_vis"] = False
+
+def _verify_df(df: pd.DataFrame, columns: List) -> bool:
+    if df is None:
+        return False
+    elif not all(column in df.columns for column in columns):
+        return False
+    else:
+        return True
 
 
 def _delete_files_from_graph(graph_id: str) -> None:
@@ -82,8 +99,8 @@ def _get_file_from_graph(file_id: str) -> Optional[File]:
 
 def _scrape_hackernews(state: WriterState) -> tuple[Any, Any]:
     stories_ids = _get_stories_ids(state)
-    posts, comments_ids = _get_posts(stories_ids)
-    comments = _get_comments(comments_ids)
+    posts, comments_ids = _get_posts(state, stories_ids)
+    comments = _get_comments(state, comments_ids)
 
     if len(posts) > 0:
         state["posts"] = pd.DataFrame(posts).sort_values(
@@ -108,10 +125,11 @@ def _get_stories_ids(state: WriterState) -> List[str]:
         return response.json()[: int(state["fetch_limit"])]
     except Exception as e:
         print(f"Failed to fetch story ids from Hacker News: {str(e)}")
+        state["message_setup"] = ""
         return []
 
 
-def _get_posts(stories_ids: List[str]) -> (List[dict], List[int]):
+def _get_posts(state: WriterState, stories_ids: List[str]) -> (List, List):
     try:
         stories_urls = [
             f"{HACKERNEWS_API_URL}/item/{story_id}.json" for story_id in stories_ids
@@ -143,10 +161,11 @@ def _get_posts(stories_ids: List[str]) -> (List[dict], List[int]):
         return posts_data, comments_ids
     except Exception as e:
         print(f"Failed to fetch stories from Hacker News: {str(e)}")
-        return ([], [])
+        state["message_setup"] = ""
+        return [], []
 
 
-def _get_comments(comments_ids: List[str]) -> List[dict]:
+def _get_comments(state: WriterState, comments_ids: List[str]) -> List[dict]:
     try:
         comments_urls = [
             f"{HACKERNEWS_API_URL}/item/{comment_id}.json"
@@ -172,6 +191,7 @@ def _get_comments(comments_ids: List[str]) -> List[dict]:
         return comments_data
     except Exception as e:
         print(f"Failed to fetch comments from Hacker News: {str(e)}")
+        state["message_setup"] = ""
         return []
 
 
@@ -194,14 +214,17 @@ def _save_results_to_csv(state: WriterState) -> None:
         state["comments"].to_csv("static/hackernews_comments.csv", index=False)
 
 
-def _upload_file_and_add_to_graph(file_path: str, graph_id: str) -> dict:
+def _upload_file_and_add_to_graph(
+    state: WriterState, file_path: str, graph_id: str
+) -> dict:
     try:
         file_id = _upload_file(file_path)
         _add_file_to_graph(graph_id, file_id)
 
         return {"file_id": file_id, "graph_id": graph_id}
     except Exception as e:
         print(f"An error while file uploading occurred: {str(e)}")
+        state["message_setup"] = ""
         return {}
 
 
@@ -220,30 +243,31 @@ def _add_file_to_graph(graph_id: str, file_id: str) -> None:
 
 def _handle_contributing_sources(state: WriterState, graph_data: dict) -> None:
     sources = graph_data.get("sources")
+    contributing_sources = {}
     if sources:
-        with wf.init_ui() as ui:
-            with ui.refresh_with("contributed_sources"):
-                for index, source in enumerate(sources):
-                    source_file = _get_file_from_graph(source["file_id"])
-                    source_snippet = source["snippet"]
-                    ui.Section(
-                        content={
-                            "title": "📄 " + source_file.name,
-                            "cssClasses": "file",
-                        },
-                        id=f"source {index}",
-                    )
-                    with ui.find(f"source {index}"):
-                        ui.Text({"text": source_snippet, "cssClasses": "file-text"})
+        for index, source in enumerate(sources):
+            source_file = _get_file_from_graph(source["file_id"])
+            source_snippet = source["snippet"]
 
+            contributing_sources.update(
+                {
+                    str(index): {
+                        "name": f"📄 {source_file.name}",
+                        "file_css": "file",
+                        "content": source_snippet,
+                        "content_css": "file-text",
+                    }
+                }
+            )
+
+        state["contributing_sources"] = contributing_sources
         state["contributing_sources_vis"] = True
         state["contributing_sources_button_text"] = "View contributing sources ▸"
 
 
 def run_report(state: WriterState) -> None:
     try:
         state["message_report"] = "%Creating report"
-        state["message_report_vis"] = True
 
         prompt = report_prompt(state["posts"], state["comments"])
         report_convo = Conversation()
@@ -255,10 +279,10 @@ def run_report(state: WriterState) -> None:
         for chunk in response:
             state["prepared_report"] += chunk["content"]
 
-        state["message_report"] = "+Creation is finished!"
-        state["message_report_vis"] = False
+        state["message_report"] = ""
     except Exception as e:
         state["prepared_report"] = "Something went wrong. Please try again!"
+        state["message_report"] = ""
         raise e
 
 
@@ -304,9 +328,13 @@ def contributing_sources_change_vis(state: WriterState) -> None:
     {
         "conversation": Conversation(
             [
-                {"role": "assistant", "content": "Ask me anything about the scraped Hacker News data."},
+                {
+                    "role": "assistant",
+                    "content": "Ask me anything about the scraped Hacker News data.",
+                },
             ],
         ),
+        "contributing_sources": {},
         "response": None,
         "file_path": "",
         "graph_name": "",
@@ -316,14 +344,10 @@ def contributing_sources_change_vis(state: WriterState) -> None:
         "contributing_sources_button_text": "View contributing sources ◂",
         "message_setup": "",
         "message_report": "",
-        "message_setup_vis": False,
-        "message_report_vis": False,
         "contributing_sources_vis": False,
         "fetch_limit": 100,
         "allow_comments": True,
     }
 )
 
-initial_state.import_frontend_module("scripts", "/static/custom.js")
 initial_state.import_stylesheet("style", "/static/custom.css")
-initial_state.call_frontend_function("scripts", "initSelectedDropdownOption", [])
diff --git a/hacker-news-social-listener/static/custom.css b/hacker-news-social-listener/static/custom.css
@@ -8,7 +8,7 @@
 }
 
 .files-list{
-    height: 84vh !important;
+    height: 79vh !important;
     overflow-y: auto !important;
 }
 
diff --git a/hacker-news-social-listener/static/custom.js b/hacker-news-social-listener/static/custom.js

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`}`
`9`	`9`
`10`	`10`	`.files-list{`
`11`		`- height: 84vh !important;`
	`11`	`+ height: 79vh !important;`
`12`	`12`	`overflow-y: auto !important;`
`13`	`13`	`}`
`14`	`14`