Get the importer working and able to build SQL

Author: peteryates

.gitignore

@@ -0,0 +1,2 @@
+olympics-database.sql
+dml/*.sql

Makefile

@@ -0,0 +1,27 @@
+db=olympics
+
+test:
+	dropdb createdb build_inserts build_types_and_tables insert
+
+insert:
+	psql ${db} < ./dml/insert_data.sql
+
+build_inserts:
+	ruby ./build/extract.rb > ./dml/insert_data.sql
+
+dropdb:
+	dropdb --if-exists ${db}
+
+createdb:
+	createdb ${db}
+
+build_types_and_tables:
+	psql ${db} < ./ddl/country.sql
+	psql ${db} < ./ddl/athlete.sql
+	psql ${db} < ./ddl/games.sql
+	psql ${db} < ./ddl/sport.sql
+	psql ${db} < ./ddl/event.sql
+	psql ${db} < ./ddl/appearance.sql
+
+build_dump:
+	cat ddl/{country,athlete,sport,games,event,appearance}.sql dml/insert_data.sql > olympics-database.sql

build/extract.rb

@@ -0,0 +1,210 @@
+require "csv"
+require "pry"
+
+output = []
+
+Q = Struct.new(:raw_value, :quotable) do
+  def value
+    raw_value == 'NA' ? 'null' : raw_value
+  end
+
+  def quotable?
+    quotable
+  end
+end
+
+class Insertable
+  def data
+    "(" + quoted_values.join(", ") + ")"
+  end
+  
+  def quoted_values
+    values.map do |qv|
+      if qv.value == 'null'
+        'null'
+      elsif qv.quotable
+        %($$#{qv.value}$$)
+      else
+        qv.value
+      end
+    end
+  end
+end
+
+class Country < Insertable
+  attr_reader :code, :name, :alternative_name
+
+  def initialize(code, name, alternative_name)
+    @code, @name, @alternative_name = code, name, alternative_name
+  end
+
+  def values
+    [
+      Q.new(code, true),
+      Q.new(name, true),
+      (
+        if alternative_name
+          Q.new(alternative_name, true)
+        else
+          Q.new("null", false)
+        end
+      )
+    ].compact
+  end
+end
+
+class Athlete < Insertable
+  attr_reader :id, :name, :sex, :height, :weight
+
+  def initialize(id, name, sex, height, weight)
+    @id     = id
+    @name   = name
+    @sex    = sex
+    @height = height
+    @weight = weight
+  end
+
+  def values
+    [Q.new(id, false), Q.new(name, true), Q.new(sex, true), Q.new(height, false), Q.new(weight, false)]
+  end
+end
+
+class Games < Insertable
+  attr_reader :code, :name, :year, :season, :city
+
+  def initialize(code, name, year, season, city)
+    @code   = code
+    @name   = name
+    @year   = year
+    @season = season
+    @city   = city
+  end
+
+  def values
+    [Q.new(code, true), Q.new(name, true), Q.new(year, false), Q.new(season, true), Q.new(city, true)]
+  end
+end
+
+class Sport < Insertable
+  attr_reader :id, :name
+
+  def initialize(id, name)
+    @id   = id
+    @name = name
+  end
+
+  def values
+    [Q.new(id, false), Q.new(name, true)]
+  end
+end
+
+class Event < Insertable
+  attr_reader :id, :sport_id, :name
+
+  def initialize(id, sport_id, name)
+    @id       = id
+    @sport_id = sport_id
+    @name     = name
+  end
+
+  def values
+    [Q.new(id, false), Q.new(sport_id, false), Q.new(name, true)]
+  end
+end
+
+class Appearance < Insertable
+  attr_reader :athlete_id, :country_code, :games_code, :event_id, :age, :medal
+
+  def initialize(athlete_id, country_code, games_code, event_id, age, medal)
+    @athlete_id   = athlete_id
+    @country_code = country_code
+    @games_code   = games_code
+    @event_id     = event_id
+    @age          = age
+    @medal        = medal
+  end
+
+  def values
+    [Q.new(athlete_id, false), Q.new(country_code, true), Q.new(games_code, true), Q.new(event_id, false), Q.new(age, false), Q.new(medal, true)]
+  end
+end
+
+def prepare(values)
+  values.map(&:data).join(",\n")
+end
+
+def create_insert(statement, values)
+  [statement, values, ";"].join("\n")
+end
+
+def extract_sex(v)
+  if v == 'M'
+    'Male'
+  elsif v == 'F'
+    'Female'
+  end
+end
+
+def games_code(year, season)
+  "#{year.to_s}-#{season.chars.first}"
+end
+
+data = CSV.read("./raw/athlete_events.csv", headers: true)
+
+countries = CSV
+  .read("./raw/noc_regions.csv", headers: true)
+  .map { |c| Country.new(c["NOC"], c["region"], c["notes"]) }
+
+output << create_insert(<<~SQL.strip, prepare(countries))
+  insert into country (code, name, alternative_name) values
+SQL
+
+athletes = data
+  .map { |r| [r['ID'], r['Name'], extract_sex(r['Sex']), r['Height'], r['Weight']] }
+  .uniq
+  .map { |r| Athlete.new(*r) }
+
+output << create_insert(<<~SQL.strip, prepare(athletes))
+  insert into athlete (id, name, sex, height, weight) values
+SQL
+
+sports = data.map { |r| r['Sport'] }.uniq.map.with_index(1) { |s, i| [i, s] }.map { |r| Sport.new(*r) }
+
+output << create_insert(<<~SQL.strip, prepare(sports))
+  insert into sport (id, name) values
+SQL
+
+sports_lookup = sports.each.with_object({}) { |s, h| h[s.name] = s.id }
+
+events = data
+  .map { |r| [r['Sport'], r['Event']] }
+  .uniq
+  .map
+  .with_index(1) { |e, i| [i, sports_lookup[e[0]], e[1]] }
+  .map { |r| Event.new(*r) }
+
+output << create_insert(<<~SQL.strip, prepare(events))
+  insert into event (id, sport_id, name) values
+SQL
+
+events_lookup = events.each.with_object({}) { |e, h| h[e.name] = e.id }
+
+games = data
+  .map { |r| [games_code(r['Year'], r['Season']), [r['Year'], r['City']].join(" - "), r['Year'], r['Season'], r['City']] }
+  .uniq { |r| [r[0]] }
+  .map { |r| Games.new(*r) }
+
+output << create_insert(<<~SQL.strip, prepare(games))
+  insert into games (code, name, year, season, city) values
+SQL
+
+appearances = data
+  .map { |r| [r['ID'], r['NOC'], games_code(r['Year'], r['Season']), events_lookup[r['Event']], r['Age'], r['Medal']] }
+  .uniq
+  .map { |r| Appearance.new(*r) }
+
+output << create_insert(<<~SQL.strip, prepare(appearances))
+  insert into appearance (athlete_id, country_code, games_code, event_id, age, medal) values
+SQL
+
+File.write("./dml/insert_data.sql", output.join("\n\n"))

ddl/appearance.sql

@@ -0,0 +1,15 @@
+drop table if exists appearance;
+
+create type medal as enum ('Gold', 'Silver', 'Bronze');
+
+create table appearance (
+	athlete_id integer not null references athlete(id),
+	country_code char(3) not null references country(code),
+	games_code char(6) not null references games(code),
+	event_id integer not null references event(id),
+	age integer null,
+	medal medal null
+);
+
+create unique index appearance_athlete_games_event on appearance(athlete_id, games_code, event_id, medal);
+create index appearance_games_athlete on appearance(games_code, athlete_id);

ddl/athlete.sql

@@ -0,0 +1,14 @@
+drop table if exists athlete;
+drop type if exists sex;
+
+create type sex as enum ('Male', 'Female');
+
+create table athlete (
+	id integer primary key,
+	name varchar(128) not null,
+	sex sex not null,
+	height integer null,
+	weight integer null
+);
+
+create index athlete_name on athlete(name);

ddl/country.sql

@@ -0,0 +1,9 @@
+drop table if exists country;
+
+create table country (
+	code char(3) primary key,
+	name varchar(32) null,
+	alternative_name varchar(32) null
+);
+
+create index country_name on country(name);

ddl/create_database.sql

@@ -0,0 +1,2 @@
+drop database if exists olympics;
+create database olympics;

ddl/event.sql

@@ -0,0 +1,10 @@
+drop table if exists event;
+
+create table event (
+	id integer primary key,
+	sport_id integer not null references sport(id),
+	name varchar(128) not null
+);
+
+create index event_name on event(name);
+create index event_sport_id on event(sport_id);

ddl/games.sql

@@ -0,0 +1,15 @@
+drop table if exists games;
+
+create type season as enum ('Summer', 'Winter');
+
+create table games (
+	code char(6) primary key,
+	name varchar(32) not null,
+	year integer not null,
+	season season not null,
+	city varchar(32) not null
+);
+
+create index games_name on games(name);
+create index games_city on games(city);
+create index games_year on games(year);

ddl/sport.sql

@@ -0,0 +1,8 @@
+drop table if exists sport;
+
+create table sport (
+	id integer primary key,
+	name varchar(32) not null
+);
+
+create index sport_name on sport(name);