Merge pull request #29 from PathwayCommons/suhyma

Implementing keyword search JS script

Author: Suhyma

.vscode/launch.json

@@ -52,6 +52,16 @@
         "--pretty"
       ],
       "console": "integratedTerminal"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "DataSearch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "program": "${workspaceFolder}/src/data-search.js",
+      "console": "integratedTerminal"
     }
   ]
 }

scripts/date.js

@@ -1,6 +1,6 @@
 import { format, sub } from 'date-fns';
 
-const startOffset = { months: 1 };
+const startOffset = { days: 1 };
 const arg = process.argv[2];
 const now = new Date();
 

src/cli.js

@@ -17,29 +17,43 @@ const writeText = async (text, file) => await writeFile(file, text);
 const printText = text => console.log(text);
 const getPrettyText = (articles, queryString, options) => prettyArticles(articles, queryString, options);
 
-async function search (queryString, options) {
-  const searcher = new Search();
+export async function search (queryString, options) {
+  try {
+    const searcher = new Search();
 
-  const articles = await getInput(options);
+    const articles = await getInput(options);
 
-  await searcher.articles(articles);
+    await searcher.articles(articles);
 
-  const res = await searcher.search(queryString, {
-    combineWith: options.strict ? 'AND' : 'OR'
-  });
+    const res = await searcher.search(queryString, {
+      combineWith: options.strict ? 'AND' : 'OR'
+    });
 
-  await sendOutput(res, options, queryString);
+    await sendOutput(res, options, queryString);
+
+    return res;
+  } catch (err) {
+    console.error(`Error in search: ${err}`);
+    throw err;
+  }
 }
 
-async function download (startDate, endDate, options) {
-  const source = options.source ?? 'biorxiv';
+export async function download (startDate, endDate, options) {
+  try {
+    const source = options.source ?? 'biorxiv';
 
-  const res = await performDownload(source, startDate, endDate);
+    const res = await performDownload(source, startDate, endDate);
 
-  await sendOutput(res, options);
+    await sendOutput(res, options);
+
+    return res;
+  } catch (err) {
+    console.error(`Error in download: ${err}`);
+    throw err;
+  }
 }
 
-async function sendOutput (res, options, queryString) {
+export async function sendOutput (res, options, queryString) {
   if (options.reverse) {
     res = res.reverse();
   }
@@ -103,4 +117,4 @@ async function main () {
   await program.parseAsync();
 }
 
-main();
+// main();

src/data-search.js

@@ -0,0 +1,61 @@
+#! /usr/bin/env node
+
+// here we will convert the search.sh script into JS
+import { format, sub } from 'date-fns';
+import fs from 'fs/promises';
+import { download, search, sendOutput } from './cli.js';
+
+const CATEGORY_ID = 'alzheimers-disease';
+const DATA_DIRECTORY = 'example-data';
+
+const MEDRXIV_SOURCE = 'medrxiv';
+const BIORXIV_SOURCE = 'biorxiv';
+
+const now = new Date();
+const startOffset = { days: 1 };
+const START_DATE = format(sub(now, startOffset), 'yyyy-MM-dd');
+const END_DATE = format(now, 'yyyy-MM-dd');
+
+const BIORXIV_FILE = `${DATA_DIRECTORY}/${END_DATE}_${BIORXIV_SOURCE}.json`;
+const MEDRXIV_FILE = `${DATA_DIRECTORY}/${END_DATE}_${MEDRXIV_SOURCE}.json`;
+const COMBINED_FILE = `${DATA_DIRECTORY}/${END_DATE}.json`;
+const OUTPUT_FILE = `${DATA_DIRECTORY}/${CATEGORY_ID}.json`;
+
+// Getting all latest articles from BiorXiv
+console.log(`Fetching from ${BIORXIV_SOURCE} between ${START_DATE} and ${END_DATE}`);
+fs.open(BIORXIV_FILE, 'w');
+const bioOptions = {
+  source: BIORXIV_SOURCE,
+  output: BIORXIV_FILE
+};
+const bioData = await download(START_DATE, END_DATE, bioOptions);
+
+// Getting all latest articles from MedrXiv
+console.log(`Fetching from ${MEDRXIV_SOURCE} between ${START_DATE} and ${END_DATE}`);
+fs.open(MEDRXIV_FILE, 'w');
+const medOptions = {
+  source: MEDRXIV_SOURCE,
+  output: MEDRXIV_FILE
+};
+const medData = await download(START_DATE, END_DATE, medOptions);
+
+// Creating a JSON with all the results, both sources combined
+console.log('Combining results...');
+fs.open(COMBINED_FILE, 'w');
+const combinedData = bioData.concat(medData);
+const combinedOptions = {
+  output: COMBINED_FILE
+};
+await sendOutput(combinedData, combinedOptions);
+
+// Search for the QUERY keyword in all the downloaded articles & compile the related articles
+const QUERY = 'alzheimer';
+fs.open(OUTPUT_FILE, 'w');
+const outputOptions = {
+  input: COMBINED_FILE,
+  output: OUTPUT_FILE
+};
+console.log(`Searching for ${QUERY}`);
+const searchHits = await search(QUERY, outputOptions);
+const numSearchHits = searchHits.length;
+console.log(`Found ${numSearchHits} hits`);