Merge pull request #133 from multinet-app/setup-dev-env

JackWilb · web-flow · commit 23fe9a076628 · 2022-06-28T16:26:34.000-06:00
Add a django command to set up the demo data
diff --git a/README.md b/README.md
@@ -8,6 +8,7 @@ This is the simplest configuration for developers to start with.
 2. Run `docker-compose run --rm django ./manage.py createsuperuser`
    and follow the prompts to create your own user. Make sure to supply an email address.
 3. Run `docker-compose run --rm django ./manage.py createarangoreadonlyuser`
+4. To hydrate the database with the demo data, run `docker-compose run --rm django ./manage.py setupdevenv <user_email from above step>`
 
 ### Run Application
 1. Run `docker-compose up`
@@ -37,6 +38,7 @@ but allows developers to run Python code on their native system.
 7. Run `./manage.py migrate`
 8. Run `./manage.py createsuperuser` and follow the prompts to create your own user. Make sure to supply an email address.
 9. Run `./manage.py createarangoreadonlyuser`
+10. To hydrate the database with the demo data, run `./manage.py setupdevenv <user_email from above step>`
 
 ### Run Application
 1.  Ensure `docker-compose -f ./docker-compose.yml up -d` is still active
diff --git a/data/boston/types/clubs.json b/data/boston/types/clubs.json
@@ -0,0 +1,4 @@
+{
+    "_key": "label",
+    "name": "label"
+}
diff --git a/data/boston/types/members.json b/data/boston/types/members.json
@@ -0,0 +1,4 @@
+{
+    "_key": "label",
+    "name": "label"
+}
diff --git a/data/boston/types/membership.json b/data/boston/types/membership.json
@@ -0,0 +1,4 @@
+{
+    "_from": "label",
+    "_to": "label"
+}
diff --git a/data/eurovis-2019/types/connections.json b/data/eurovis-2019/types/connections.json
@@ -0,0 +1,5 @@
+{
+    "_key": "label",
+    "_from": "label",
+    "_to": "label"
+}
diff --git a/data/eurovis-2019/types/people.json b/data/eurovis-2019/types/people.json
@@ -0,0 +1,14 @@
+{
+    "_key": "label",
+    "followers_count": "number",
+    "query_tweet_count": "number",
+    "friends_count": "number",
+    "statuses_count": "number",
+    "listed_count": "number",
+    "favourites_count": "number",
+    "count_followers_in_query": "number",
+    "screen_name": "label",
+    "profile_image_url": "label",
+    "influential": "boolean",
+    "original": "boolean"
+}
diff --git a/data/miserables/types/characters.json b/data/miserables/types/characters.json
@@ -0,0 +1,5 @@
+{
+    "_key": "label",
+    "name": "label",
+    "group": "category"
+}
diff --git a/data/miserables/types/relationships.json b/data/miserables/types/relationships.json
@@ -0,0 +1,5 @@
+{
+    "_from": "label",
+    "_to": "label",
+    "value": "number"
+}
diff --git a/data/movies/types/edgelist.json b/data/movies/types/edgelist.json
@@ -0,0 +1,4 @@
+{
+    "actors": "label",
+    "films": "label"
+}
diff --git a/data/movies/types/films.json b/data/movies/types/films.json
@@ -0,0 +1,21 @@
+{
+    "Title": "label",
+    "Alternate title": "label",
+    "Genre": "category",
+    "Primary Source Verification": "label",
+    "Notes": "label",
+    "Production Company": "label",
+    "Production date": "label",
+    "Distribution date": "label",
+    "Length (reels)": "label",
+    "Actors": "label",
+    "Directors": "label",
+    "Producers": "label",
+    "Camerawork": "label",
+    "Attachments": "label",
+    "Source": "label",
+    "Writer": "label",
+    "Otherwise Involved": "label",
+    "Link to media": "label",
+    "Distributor": "label"
+}
diff --git a/data/movies/types/nodelist.json b/data/movies/types/nodelist.json
@@ -0,0 +1,4 @@
+{
+    "id": "label",
+    "type": "category"
+}
diff --git a/data/openflights/types/airports.json b/data/openflights/types/airports.json
@@ -0,0 +1,16 @@
+{
+    "_key": "label",
+    "name": "label",
+    "city": "label",
+    "country": "label",
+    "IATA": "label",
+    "ICAO": "label",
+    "latitude": "number",
+    "longitude": "number",
+    "altitude": "number",
+    "timezone": "label",
+    "DST": "label",
+    "tz database time": "label",
+    "type": "category",
+    "source": "category"
+}
diff --git a/data/openflights/types/routes.json b/data/openflights/types/routes.json
@@ -0,0 +1,11 @@
+{
+    "airline": "category",
+    "airline ID": "category",
+    "source airport": "label",
+    "_from": "label",
+    "destination airport": "label",
+    "_to": "label",
+    "codeshare": "boolean",
+    "stops": "number",
+    "equipment": "label"
+}
diff --git a/multinet/api/management/commands/setupdevenv.py b/multinet/api/management/commands/setupdevenv.py
@@ -0,0 +1,129 @@
+import csv
+from io import StringIO
+import json
+import os
+from pathlib import Path
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+from multinet.api.models import Network, Table, TableTypeAnnotation, Workspace
+from multinet.api.tasks.upload.csv import process_row
+
+
+class Command(BaseCommand):
+    help = 'Sets up a standard development environment with example workspaces and tables.'
+
+    def create_workspace_if_not_exists(self, workspace_name):
+        try:
+            # Check if workspace exists, if so remove it
+            if Workspace.objects.filter(name=workspace_name).exists():
+                self.stdout.write(
+                    self.style.SUCCESS(f'{workspace_name} already exists, removing to recreate')
+                )
+                Workspace.objects.filter(name=workspace_name).delete()
+
+            # Create the workspace
+            Workspace.objects.create(name=workspace_name, public=True, owner=self.user)
+            self.stdout.write(self.style.SUCCESS(f'{workspace_name} created'))
+
+        except Exception as e:
+            self.stderr.write(self.style.ERROR(str(e)))
+
+    def create_tables_for_workspace(self, workspace_name, edge_table_name):
+        try:
+            # Get the paths for all the data objects to upload
+            data_dir_path = Path(settings.BASE_DIR) / 'data' / workspace_name
+            csv_paths = list(data_dir_path.glob('*.csv'))
+            workspace = Workspace.objects.get(name=workspace_name)
+
+            for csv_path in csv_paths:
+                filename = os.path.splitext(csv_path.name)[0]
+
+                # Check if table with name exists, if so delete
+                if Table.objects.filter(workspace=workspace, name=filename).exists():
+                    self.stdout.write(
+                        self.style.SUCCESS(
+                            f'{workspace_name}/{filename} already exists, removing to recreate'
+                        )
+                    )
+                    Table.objects.filter(workspace=workspace, name=filename).delete()
+
+                # Open the types file and read the types in
+                types_path = data_dir_path / 'types' / f'{filename}.json'
+                with types_path.open('rb') as f:
+                    columns = json.load(f)
+
+                # Open the csv and read in the rows
+                with csv_path.open('rb') as f:
+                    csv_rows = list(csv.DictReader(StringIO(f.read().decode('utf-8'))))
+
+                # Process csv rows with the type annotations
+                for i, row in enumerate(csv_rows):
+                    csv_rows[i] = process_row(row, columns)
+
+                # Create the table
+                new_table = Table.objects.create(
+                    workspace=workspace, name=filename, edge=filename == edge_table_name
+                )
+
+                # Put the rows into the table
+                new_table.put_rows(csv_rows)
+
+                # Create type annotations
+                TableTypeAnnotation.objects.bulk_create(
+                    [
+                        TableTypeAnnotation(table=new_table, column=col_key, type=col_type)
+                        for col_key, col_type in columns.items()
+                    ]
+                )
+
+                self.stdout.write(self.style.SUCCESS(f'{workspace_name}/{filename} created'))
+
+        except Exception as e:
+            self.stderr.write(self.style.ERROR(str(e)))
+
+    def create_network_in_workspace(self, workspace_name, edge_table_name, node_table_names):
+        try:
+            workspace = Workspace.objects.get(name=workspace_name)
+
+            # Create the network
+            Network.create_with_edge_definition(
+                workspace_name, workspace, edge_table_name, node_table_names
+            )
+            self.stdout.write(self.style.SUCCESS(f'{workspace_name} network created'))
+
+        except Exception as e:
+            self.stderr.write(self.style.ERROR(str(e)))
+
+    def add_arguments(self, parser):
+        parser.add_argument('email', type=str)
+
+    def handle(self, *args, **options):
+        # Check that owner exists
+        self.user = User.objects.filter(email=options['email']).first()
+        if self.user is None:
+            raise Exception('User did not exist')
+
+        # Create the workspaces
+        self.create_workspace_if_not_exists('boston')
+        self.create_workspace_if_not_exists('eurovis-2019')
+        self.create_workspace_if_not_exists('miserables')
+        self.create_workspace_if_not_exists('movies')
+        self.create_workspace_if_not_exists('openflights')
+
+        # Create the tables
+        self.create_tables_for_workspace('boston', 'membership')
+        self.create_tables_for_workspace('eurovis-2019', 'connections')
+        self.create_tables_for_workspace('miserables', 'relationships')
+        self.create_tables_for_workspace('openflights', 'routes')
+
+        # Create the networks from the tables
+        self.create_network_in_workspace('boston', 'membership', ['clubs', 'members'])
+        self.create_network_in_workspace('eurovis-2019', 'connections', ['people'])
+        self.create_network_in_workspace('miserables', 'relationships', ['characters'])
+        self.create_network_in_workspace('openflights', 'routes', ['airports'])
+
+        # TODO: Include movies dataset here once new network creation API
+        # is accessible internally from the API

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +    "_key": "label",
 +    "name": "label"
 +}