Merge pull request #30 from PathwayCommons/suhyma

Finalizing data-search script & cron job

Author: Suhyma

.vscode/launch.json

@@ -4,6 +4,19 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Test",
+      "request": "launch",
+      "runtimeArgs": [
+        "run-script",
+        "test"
+      ],
+      "runtimeExecutable": "npm",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "type": "node"
+    },
     {
       "name": "NPM watch",
       "runtimeVersion": "16.17.0",

example-data/data-config.json

@@ -27,7 +27,7 @@
     "fix": "eslint ./src --fix",
     "lint": "eslint ./src",
     "clean": "rimraf dist src/dashboard/build",
-    "data": "echo \"Hello World!\""
+    "data": "node src/data-search.js"
   },
   "author": "Pathway Commons",
   "license": "MIT",
@@ -55,6 +55,7 @@
     "date-fns": "^2.29.3",
     "eventemitter3": "^4.0.7",
     "get-stdin": "^9.0.0",
+    "lodash": "^4.17.21",
     "minisearch": "^5.0.0",
     "node-fetch": "^3.2.6",
     "preact": "^10.11.0",

package-lock.json

@@ -12,29 +12,45 @@ import { prettyArticles } from './pretty.js';
 const readFile = promisify(fs.readFile);
 const formatJSON = obj => JSON.stringify(obj, null, 2);
 const printFormattedJSON = obj => console.log(formatJSON(obj));
-const writeFormattedJSON = async (obj, file) => await writeFile(file, formatJSON(obj));
+export const writeFormattedJSON = async (obj, file) => await writeFile(file, formatJSON(obj));
 const writeText = async (text, file) => await writeFile(file, text);
 const printText = text => console.log(text);
 const getPrettyText = (articles, queryString, options) => prettyArticles(articles, queryString, options);
 
 export async function search (queryString, options) {
-  try {
-    const searcher = new Search();
+  const searcher = new Search();
 
+  if (options.array) {
+    const articles = options.array;
+    await searcher.articles(articles);
+  } else {
     const articles = await getInput(options);
-
     await searcher.articles(articles);
+  }
 
-    const res = await searcher.search(queryString, {
-      combineWith: options.strict ? 'AND' : 'OR'
-    });
+  const res = await searcher.search(queryString, {
+    combineWith: options.strict ? 'AND' : 'OR'
+  });
 
-    await sendOutput(res, options, queryString);
+  const source = options.source ?? 'biorxiv';
 
-    return res;
-  } catch (err) {
-    console.error(`Error in search: ${err}`);
-    throw err;
+  if (source === 'biorxiv' || source === 'medrxiv') {
+    try {
+      const formattedRes = await formatData(res);
+      await sendOutput(formattedRes, options, queryString);
+      return formattedRes;
+    } catch (err) {
+      console.error(`Error in search: ${err}`);
+      throw err;
+    }
+  } else {
+    try {
+      await sendOutput(res, options, queryString);
+      return res;
+    } catch (err) {
+      console.error(`Error in search: ${err}`);
+      throw err;
+    }
   }
 }
 
@@ -86,6 +102,24 @@ async function getInput (options) {
   }
 }
 
+async function formatData (dataArray) {
+  try {
+    const formattedData = dataArray.map(article => ({
+      paperId: article.doi,
+      doi: article.doi,
+      title: article.title,
+      journal: article.server,
+      date: article.date,
+      brief: null,
+      authors: article.authors
+    }));
+    return formattedData;
+  } catch (err) {
+    console.error(`Error in formatting data: ${err}`);
+    throw err;
+  }
+}
+
 async function main () { // eslint-disable-line no-unused-vars
   (program
     .name('hyper-recent')

package.json

@@ -1,61 +1,48 @@
 #! /usr/bin/env node
-
-// here we will convert the search.sh script into JS
+import _ from 'lodash';
+import fs from 'fs';
 import { format, sub } from 'date-fns';
-import fs from 'fs/promises';
-import { download, search, sendOutput } from './cli.js';
-
-const CATEGORY_ID = 'alzheimers-disease';
-const DATA_DIRECTORY = 'example-data';
-
-const MEDRXIV_SOURCE = 'medrxiv';
-const BIORXIV_SOURCE = 'biorxiv';
-
-const now = new Date();
-const startOffset = { days: 1 };
-const START_DATE = format(sub(now, startOffset), 'yyyy-MM-dd');
-const END_DATE = format(now, 'yyyy-MM-dd');
-
-const BIORXIV_FILE = `${DATA_DIRECTORY}/${END_DATE}_${BIORXIV_SOURCE}.json`;
-const MEDRXIV_FILE = `${DATA_DIRECTORY}/${END_DATE}_${MEDRXIV_SOURCE}.json`;
-const COMBINED_FILE = `${DATA_DIRECTORY}/${END_DATE}.json`;
-const OUTPUT_FILE = `${DATA_DIRECTORY}/${CATEGORY_ID}.json`;
-
-// Getting all latest articles from BiorXiv
-console.log(`Fetching from ${BIORXIV_SOURCE} between ${START_DATE} and ${END_DATE}`);
-fs.open(BIORXIV_FILE, 'w');
-const bioOptions = {
-  source: BIORXIV_SOURCE,
-  output: BIORXIV_FILE
-};
-const bioData = await download(START_DATE, END_DATE, bioOptions);
-
-// Getting all latest articles from MedrXiv
-console.log(`Fetching from ${MEDRXIV_SOURCE} between ${START_DATE} and ${END_DATE}`);
-fs.open(MEDRXIV_FILE, 'w');
-const medOptions = {
-  source: MEDRXIV_SOURCE,
-  output: MEDRXIV_FILE
-};
-const medData = await download(START_DATE, END_DATE, medOptions);
-
-// Creating a JSON with all the results, both sources combined
-console.log('Combining results...');
-fs.open(COMBINED_FILE, 'w');
-const combinedData = bioData.concat(medData);
-const combinedOptions = {
-  output: COMBINED_FILE
-};
-await sendOutput(combinedData, combinedOptions);
-
-// Search for the QUERY keyword in all the downloaded articles & compile the related articles
-const QUERY = 'alzheimer';
-fs.open(OUTPUT_FILE, 'w');
-const outputOptions = {
-  input: COMBINED_FILE,
-  output: OUTPUT_FILE
-};
-console.log(`Searching for ${QUERY}`);
-const searchHits = await search(QUERY, outputOptions);
-const numSearchHits = searchHits.length;
-console.log(`Found ${numSearchHits} hits`);
+import { download } from './download.js';
+import { Search } from './search.js';
+import { writeFormattedJSON } from './cli.js';
+
+/**
+ * Download preprint data from BiorXiv and MedrXiv servers and perform search for preprints in each topic.
+ * @returns {collection}, a JSON array of search results for each set topic.
+ */
+export async function getData () {
+  // Set dates for past month
+  const now = new Date();
+  const startOffset = { months: 1 };
+  const start = format(sub(now, startOffset), 'yyyy-MM-dd');
+  const end = format(now, 'yyyy-MM-dd');
+
+  // Reading config file for list of topics
+  const config = JSON.parse(fs.readFileSync('example-data/data-config.json'));
+
+  // Download all recent papers & combine the arrays
+  const data = await Promise.all([
+    download('biorxiv', start, end),
+    download('medrxiv', start, end)
+  ]);
+
+  const articles = _.flatten(data);
+
+  // Search using list of topic objects from config
+  const searcher = new Search();
+
+  await searcher.articles(articles);
+  const doSearches = async config => {
+    const { keywords } = config;
+    const papers = await searcher.search(keywords, {
+      combineWith: 'AND'
+    });
+    return _.assign({}, config, { papers });
+  };
+  const collection = await Promise.all(config.map(doSearches));
+
+  // Output all search result papers into data.json
+  await writeFormattedJSON(collection, 'example-data/data.json');
+}
+
+getData();