feat: Fix issue pull limits and add filter out done (#90)

Signed-off-by: Roger Barker <roger.barker@swirldslabs.com>

Author: rbarker-dev

src/static_site_generator.py

@@ -15,7 +15,7 @@ def __init__(self):
             'tasks': []
         }
 
-    def generate_site(self, data:list=None, projects:list[str]=None, output_file='./_site/index.html') -> None:
+    def generate_site(self, data:list=None, filter_done:bool=False, projects:list[str]=None, output_file='./_site/index.html') -> None:
         logging.info("Generating Static Site")
         env = Environment(loader=FileSystemLoader('templates'))
         template = env.get_template('kaban_board.html')
@@ -34,9 +34,14 @@ def generate_site(self, data:list=None, projects:list[str]=None, output_file='./
         for task in self.GENERATOR_DATA["tasks"]:
             if "status" not in task:
               task["status"] = "NONE"
+            elif task["status"].upper() == "DONE" and filter_done == True:
+                # remove the task from the list
+                self.GENERATOR_DATA["tasks"].remove(task)
+                continue
             else:
               task["status"] = task["status"].upper()
             statuses.add(task["status"])
+        print(f"[bold cyan]Adding {len(self.GENERATOR_DATA["tasks"])} tasks to the site[/bold cyan]")
 
         # Extract unique statuses from the tasks list
         unique_statuses:list[str] = sorted(statuses)

src/version2config.py

@@ -37,6 +37,13 @@ def init_parser(self):
           help="The temporary directory to store the json data files"
         )
 
+        parser.add_argument(
+            "--filter-done",
+            dest="filter_done",
+            action="store_true",
+            help="Filter out all issues and PRs that are marked as done"
+        )
+
         parser.add_argument(
             "--include-project",
             dest="include_project",
@@ -176,6 +183,7 @@ def init_parser(self):
         self.exclude_label = parsed_args.exclude_label
         self.publish_board = parsed_args.publish_board
         self.exclude_team = parsed_args.exclude_team
+        self.filter_done = parsed_args.filter_done if parsed_args.filter_done else False
 
     def init_logger(self):
         logging.basicConfig(level=self.LOG_LEVEL, format=self.LOG_FORMAT)
@@ -187,6 +195,7 @@ def display_config(self):
         logging.info("Configuration:")
         logging.info(f"Output File: {self.output_file}")
         logging.info(f"Temporary Directory: {self.temp_dir}")
+        logging.info(f"Filter Done: {self.filter_done}")
         logging.info(f"Include Project: {self.include_project}")
         logging.info(f"Include User: {self.include_user}")
         logging.info(f"Include Repository: {self.include_repository}")

src/version2query.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from ghapi.all import GhApi
 from rich import print
+from time import sleep
 from .queryFilter import QueryFilter
 
 class Version2Query:
@@ -130,7 +131,9 @@ def fetch_project_items(self, projects:list[dict]) -> bool:
       # use os.system instead of subprocess.run or subprocess.popen to avoid creating a new
       # process space where we would need to copy environment variables into it (possibly
       # exposing things like GH_TOKEN or other environment variables)
-      os.system( f'gh project item-list --owner "{org}" {number} --format json > "{out_path}"')
+      # The default limit on the gh project item-list command is 30 items. We need to update
+      # the limit to 1000 to get all items in the project.
+      os.system( f'gh project item-list --owner "{org}" {number} --limit 1000 --format json > "{out_path}"')
 
       if not out_path.exists():
           print(f"[red]Failed to fetch items for project: {title} (Org: {org}, Project: {number})[/red]")
@@ -148,6 +151,7 @@ def consolidate_items(self) -> bool:
         with open(file) as f:
           data = json.load(f)
           items = data.get("items", [])
+          print(f"[green]Found {len(items)} items in {file}[/green]")
           all_items.extend(items)
     except Exception as e:
       print(f"[red]Error consolidating items: {e}[/red]")

version2.py

@@ -46,7 +46,7 @@ def main():
     with open(output_file, 'r') as f:
       data = json.load(f)
 
-    ss_gen.generate_site(data=data, projects=filters["include_projects"])
+    ss_gen.generate_site(data=data, filter_done=config.filter_done, projects=filters["include_projects"])
 
 if __name__ == "__main__":
     main()