Merge pull request #28 from PathwayCommons/debug-scripts

Updated data scripts

Author: jvwong

scripts/recommendations.sh

@@ -18,12 +18,14 @@ PAPER_RECOMMENDATIONS_PATH=papers/forpaper/
 PAPER_RECOMMENDATIONS_LIMIT=500
 PAPER_RECOMMENDATIONS_FIELDS=title,year,externalIds,venue,authors,abstract
 
+printf "START: recommendations \n\n"
+
 for i in ${!CATEGORY_IDS[@]}; do
   CATEGORY_ID=${CATEGORY_IDS[$i]}
   PAPER_ID=${PAPER_IDS[$i]}
   RECOMMENDATION_URL="${API_BASE_URL}${PAPER_RECOMMENDATIONS_API_PATH}${API_VERSION_PATH}${PAPER_RECOMMENDATIONS_PATH}${PAPER_ID}?limit=${PAPER_RECOMMENDATIONS_LIMIT}&fields=${PAPER_RECOMMENDATIONS_FIELDS}"
 
-  echo "Fetching from Semantic Scholar ${RECOMMENDATIONS_API_PATH}"
+  echo "Fetching from Semantic Scholar ${RECOMMENDATION_URL}"
   echo "Paper: ${PAPER_ID}"
   echo "CATEGORY_ID: ${CATEGORY_ID}"
 
@@ -35,7 +37,7 @@ for i in ${!CATEGORY_IDS[@]}; do
 
   selectedPapers=$(
     echo ${recommendedPapers} | jq '[ .[] |
-    select((.venue == "bioRxiv" or (.venue == "medRxiv")) and (.year == 2022)) |
+    select((.venue == "bioRxiv" or (.venue == "medRxiv")) and (.year >= 2022)) |
     .paperId
     ] |
     .[0:'${MAX_PAPERS}']'
@@ -84,3 +86,4 @@ for i in ${!CATEGORY_IDS[@]}; do
   echo ']' >> ${DATA_DIRECTORY}/${CATEGORY_ID}.json
 done
 
+printf "END: recommendations \n\n"

scripts/search.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 CATEGORY_ID="alzheimers-disease"
 DATA_DIRECTORY="../example-data"
@@ -9,28 +10,21 @@ BIORXIV_SOURCE="biorxiv"
 START_DATE=$(node date.js start)
 END_DATE=$(node date.js end)
 
+printf "START: search \n\n"
+
 echo "Fetching from ${BIORXIV_SOURCE} between ${START_DATE} and ${END_DATE}"
-biorxivPapers=$(node ../src/cli.js download --output="${DATA_DIRECTORY}/${END_DATE}_${BIORXIV_SOURCE}.json" --source=${BIORXIV_SOURCE} ${START_DATE} ${END_DATE})
-numPapersRawBiorxiv=$(echo ${biorxivPapers} | jq 'length')
-echo "numPapersRawBiorxiv: ${numPapersRawBiorxiv}"
+node ../src/cli.js download --output="${DATA_DIRECTORY}/${END_DATE}_${BIORXIV_SOURCE}.json" --source=${BIORXIV_SOURCE} ${START_DATE} ${END_DATE}
 
 echo "Fetching from ${MEDRXIV_SOURCE} between ${START_DATE} and ${END_DATE}"
-medrxivPapers=$(node ../src/cli.js download --output="${DATA_DIRECTORY}/${END_DATE}_${MEDRXIV_SOURCE}.json" --source=${MEDRXIV_SOURCE} ${START_DATE} ${END_DATE})
-numPapersRawMedrxiv=$(echo ${medrxivPapers} | jq 'length')
-echo "numPapersRawMedrxiv: ${numPapersRawMedrxiv}"
+node ../src/cli.js download --output="${DATA_DIRECTORY}/${END_DATE}_${MEDRXIV_SOURCE}.json" --source=${MEDRXIV_SOURCE} ${START_DATE} ${END_DATE}
 
 echo "Combining results..."
-combined=$(jq --slurp '[.[][]]' ${DATA_DIRECTORY}/${END_DATE}_*.json)
-# rm "${DATA_DIRECTORY}/${END_DATE}_${BIORXIV_SOURCE}.json"
-# rm "${DATA_DIRECTORY}/${END_DATE}_${MEDRXIV_SOURCE}.json"
-
 DATA_FILE="${DATA_DIRECTORY}/${END_DATE}.json"
-echo ${combined} | jq > ${DATA_FILE}
-
+jq --slurp '[.[][]]' ${DATA_DIRECTORY}/${END_DATE}_*.json > ${DATA_FILE}
 
 QUERY="alzheimer"
 OUTPUT_FILE="${DATA_DIRECTORY}/${CATEGORY_ID}.json"
-echo "Searching for ${CATEGORY_ID}"
+echo "Searching for ${QUERY}"
 searchHits=$(node ../src/cli.js search --strict --input=${DATA_FILE} ${QUERY})
 numSearchHits=$(echo ${searchHits} | jq 'length')
 echo "Found ${numSearchHits} hits"
@@ -48,5 +42,6 @@ collection=$(
     }]'
   )
 
-echo ${collection} | jq > ${OUTPUT_FILE}
-# rm "${DATA_FILE}"
+echo ${collection} | jq '.' > ${OUTPUT_FILE}
+
+printf "END: search \n\n"