Merge branch 'dev' into main

Author: MaoSong2022

.gitignore

@@ -178,3 +178,6 @@ TexSoup/tests/
 *.tex
 *.aux
 *.pdf
+
+# jupyter notebook
+*.ipynb

batch_process.py

@@ -4,13 +4,16 @@
 import multiprocessing
 from typing import List
 from uuid import uuid4
+import pandas as pd
 
 from vrdu import logger
 from vrdu import utils
 from main import process_one_file
 
 log_file = str(uuid4()) + ".log"
-log = logger.setup_app_level_logger(file_name=log_file, level="INFO", mode="a")
+log = logger.setup_app_level_logger(file_name=log_file, level="INFO")
+
+database = "data/processed_paper_database.csv"
 
 
 def filter_tex_files(tex_files: List[str], main_path: str = None) -> List[str]:
@@ -39,38 +42,57 @@ def filter_tex_files(tex_files: List[str], main_path: str = None) -> List[str]:
             log.debug(f"failed to read tex file: {tex_file}")
             continue
 
+    log.info(f"Before filtering, Found {len(result)} tex files")
+    if os.path.exists(database):
+        df = pd.read_csv(database)
+        processed_papers = set(df[df["status"] != "processing"]["path"])
+        result = [x for x in result if os.path.dirname(x) not in processed_papers]
+
+    log.info(f"After filtering, Found {len(result)} tex files")
     return result
 
 
-def process_one_category(path, cpu_count, category):
-    category_path = os.path.join(path, category)
-    log.info(f"path to raw data: {category_path}")
+def process_one_discpline(path: str, cpu_count: int, discpline: str) -> None:
+    discpline_path = os.path.join(path, discpline)
+    log.info(f"path to raw data: {discpline_path}")
     log.info(f"Using cpu counts: {cpu_count}")
-    tex_files = utils.extract_all_tex_files(category_path)
-    tex_files = filter_tex_files(tex_files, category_path)
+    tex_files = utils.extract_all_tex_files(discpline_path)
+    tex_files = filter_tex_files(tex_files, discpline_path)
     log.info(f"Found {len(tex_files)} tex files")
 
     try:
         with multiprocessing.Pool(cpu_count) as pool:
             pool.map(process_one_file, tex_files)
         # save log file
     except Exception:
-        log.exception(f"[VRDU] category: {category}, failed to process.")
+        log.exception(f"[VRDU] discpline: {discpline}, failed to process.")
     finally:
         # save the process log
-        shutil.move(log_file, f"batch_process_{category}.log")
+        log.info(f"[VRDU] discpline: {discpline}, finished processing.")
+        shutil.move(log_file, f"data/batch_process_{discpline}.log")
 
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-p", "--path", type=str, required=True)
-    parser.add_argument("-c", "--cpu_count", type=int, required=True)
-    parser.add_argument("-t", "--category", type=str, required=False)
+    parser.add_argument(
+        "-p", "--path", type=str, required=True, help="path to raw data"
+    )
+    parser.add_argument(
+        "-c",
+        "--cpu_count",
+        type=int,
+        required=True,
+        help="cpu count for multiprocessing",
+    )
+    parser.add_argument(
+        "-t", "--discpline", type=str, required=True, help="discpline to process"
+    )
     args = parser.parse_args()
-    path, cpu_count, category = args.path, args.cpu_count, args.category
+    path, cpu_count, discpline = args.path, args.cpu_count, args.discpline
 
-    categories = [category] if category is not None else utils.get_all_categories()
+    log.info(f"[VRDU] discpline: {discpline}, start to process.")
+    process_one_discpline(path, cpu_count, discpline)
 
-    for category in categories:
-        log.info(f"Processing single category: {category}")
-        process_one_category(path, cpu_count, category)
+
+if __name__ == "__main__":
+    main()

scripts/category_count.csv data/discpline_info.csvscripts/category_count.csv renamed to data/discpline_info.csv

@@ -1,4 +1,4 @@
-categories,count
+discpline,count
 cs.LG,68
 cs.CV,56
 quant-ph,40
@@ -122,3 +122,33 @@ cs.DM,1
 cs.SC,1
 physics.med-ph,1
 q-fin.GN,1
+q-bio.OT,1
+nlin.CG,1
+cs.AR,1
+cs.OH,1
+nlin.PS,1
+cs.GL,1
+q-bio.TO,1
+physics.pop-ph,1
+q-fin.CP,1
+q-bio.MN,1
+q-bio.SC,1
+math.AC,1
+physics.ao-ph,1
+cs.MM,1
+physics.ed-ph,1
+q-fin.EC,1
+cs.SY,1
+nlin.CD,1
+cs.PL,1
+cs.PF,1
+cs.OS,1
+q-bio.GN,1
+q-fin.MF,1
+cs.MS,1
+math.KT,1
+cs.DL,1
+econ.GN,1
+q-fin.PM,1
+stat.OT,1
+astro-ph,1