works!

Author: richiverse

.gitignore

@@ -1,3 +1,5 @@
+# zappa
+zappa_settings.json
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -23,6 +25,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+*.pyc
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

README.md

@@ -1,2 +1,79 @@
 # sql-lambda
-Register encrypted backends with JOSE and dynamodb and execute sql with lambda
+Register backends with AWS Dynamodb and execute sql queries with API Gateway,
+which talks to Lambda.
+
+## Usage
+
+Once you've obtained an API_KEY from your administrator or deploying the api, I recommend using [httpie](httpie.org) or [requests](http://docs.python-requests.org/en/master/) in [jupyter notebook](https://github.com/jupyter/notebook) to call the endpoints.
+
+#### http commands in the terminal
+```bash
+# To list all routes this service contains
+http https://xxxxxx.execute-api.us-east-1.amazonaws.com/dev/sql x-api-key:$API_KEY
+```
+
+#### requests library in python
+
+```python
+from os import environ as env
+
+import requests
+
+url = 'https://xxxxxx.execute-api.us-east-1.amazonaws.com/dev/sql'
+headers = {'x-api-key': env['API_KEY']}
+response = requests.get(url, headers=headers) 
+print(response.json)
+
+```
+
+## Prerequisites for deployment
+
+* AWS profile name in ~/.aws/credentials and valid credentials for deploying to Zappa.
+* Update zappa_settings-example.json for your environments
+* You will need an API Gateway key for encryption/decryption. 
+* You will need a service account or user credentials in order to connect.
+* Obviously do not share your api key with anyone as they can access whatever you can access with it.
+
+I **STRONGLY** advise you to use `api_key_required: true` as well as [IP restriction in IAM](http://benfoster.io/blog/aws-api-gateway-ip-restrictions) to limit usage of this API to within your VPN.
+
+## Deploying the api
+Assuming you're already familiar with zappa deployments here.
+
+You will need to build and run these commands in the provided Dockerfile outside of Linux.
+
+Docker based install instructions [here](https://github.com/danielwhatmuff/zappa#using-exported-aws_default_region-aws_secret_access_key-and-aws_access_key_id-env-vars)
+
+Linux 
+```
+cd ./sql/api
+virtualenv venv
+. ./venv/bin/activate
+pip install -r requirements.txt
+zappa deploy dev
+```
+
+## Deploying the registration app
+
+
+Please note to use a separate virtualenv for the api.
+```bash
+deactivate # We are using a separate virtualenv as these reqs/dockerfile are heavier.
+cd ./sql/app
+# same as above in a separate virtualenv
+```
+
+
+## First Step for DBA: Register your backend(s)
+
+1. Go to your deployed app URL that zappa provides
+2. Click Register backend
+3. Follow the instructions and use the api key you got from zappa when deploying the api.
+
+You should see a new table in DynamoDB with your encrypted credentials.
+
+## Test your connection
+
+```bash
+http <your-api>/<your-env>/sql/view/<your-registered-backend-name> x-api-key:$API_KEY \
+sql=="select 1"
+```

api/Dockerfile

@@ -0,0 +1,26 @@
+FROM lambci/lambda:build
+
+MAINTAINER "rich fernandez" <richardfernandeznyc@gmail.com>
+
+
+COPY docker/yum.conf /etc/yum.conf
+COPY docker/bashrc /root/.bashrc
+
+RUN yum clean all && \
+    yum -y install python27-devel python27-pip gcc \
+      vim \
+      # Your specific system requirements below
+      postgresql postgresql-devel freetds-dev 
+
+WORKDIR /var/task
+COPY . /var/task
+
+RUN pip install -U pip
+RUN pip install -U virtualenv
+
+RUN virtualenv venv && \
+    . ./venv/bin/activate && \
+    pip install -U pip && \
+    pip install -U zappa && \
+    pip install -r requirements.txt 
+    # pip install pandas sqlalchemy

api/api.py

@@ -0,0 +1,166 @@
+#! /usr/bin/env python2
+"""SQL as a Service"""
+from ast import literal_eval
+from functools import partial
+from json import loads
+from os import environ as env
+from traceback import format_exc
+
+from flask import Flask, request, jsonify
+from sqlalchemy.exc import ProgrammingError, OperationalError
+from pynamodb.exceptions import (
+    PynamoDBConnectionError,
+    QueryError,
+    GetError,
+    DeleteError,
+    DoesNotExist
+)
+
+from engine import SqlClient
+from jinja2 import Environment
+from models import Backend
+from middleware import list_routes
+from utils import decrypt
+
+api = Flask(__name__)
+
+def raise_if_not_exists(model):
+    if not model.exists():
+        raise DoesNotExist()
+
+@api.errorhandler(Exception)
+def exception_handler(error):
+    # type: (Exception) -> Exception
+    """Show uncaught exceptions.
+
+    Args:
+        error
+
+    Raises:
+        Exception
+        """
+    raise Exception(format_exc())
+
+@api.route('/')
+def list_api_routes():
+    """List all endpoints"""
+    return jsonify(list_routes(api))
+
+@api.route('/backend/list')
+def get_backends():
+    raise_if_not_exists(Backend)
+    return ''.join({item.name:
+                    item.description
+                    for item in Backend.scan()})
+
+@api.route('/register/<backend>', methods=['POST'])
+def register_backend(backend):
+    """See client.py to generate new backends."""
+
+    if not Backend.exists():
+        print('Creating table Backend for item: %s' % backend)
+        Backend.create_table(wait=True)
+
+    json = request.json
+    credentials = json['credentials']
+    description = json['description']
+
+    new_backend = Backend(
+        name=backend,
+        description=description,
+        credentials=credentials,
+    )
+    new_backend.save()
+    return jsonify("""{backend}: {description} has been registered.
+              Run ./scripts/sqlcmd to run SQL
+            """.format(backend=backend, description=description)
+    )
+
+
+@api.route('/backend/<backend>', methods=['DELETE'])
+def delete_backend(backend):
+    raise_if_not_exists(Backend)
+    items = Backend.query('name', name__eq=backend)
+    for item in items:
+        try:
+            item.delete()
+        except DeleteError as err:
+            return err
+    return jsonify("%s has been deleted" % backend)
+
+
+@api.route('/view/<backend>')
+def view_sql(backend):
+    """View data from database.
+
+    By default, thise route runs in autocommit false.
+
+    Args:
+        sql: Valid sql for your backend
+        sql_params: Optional jinja2 params
+
+    Returns:
+        json
+    """
+    key = request.headers['x-api-key']
+    args = request.args
+    sql = args['sql']
+    sql_params = args.get('sql_params', '{}')
+    rendered_sql = Environment().from_string(
+        sql).render(dict(**literal_eval(sql_params)))
+    raise_if_not_exists(Backend)
+    backend_results = {item.name: item.credentials
+                       for item in Backend.query('name', name__eq=backend)}
+    credentials = backend_results[backend]
+
+    backend_client = SqlClient(decrypt(credentials, key=key), autocommit=False)
+    viewer = backend_client.sql_viewer()
+    try:
+        results = viewer(rendered_sql)
+    except ProgrammingError as err:
+        return str(err), 400
+    except OperationalError as err:
+        return str(err), 400
+    except Exception as err:
+        return str(err), 400
+    return jsonify(results.to_json(orient='records'))
+
+
+@api.route('/execute/<backend>', methods=['POST'])
+def execute_sql(backend):
+    """Modify data from the database.
+
+    Args:
+        sql: valid sql for your backend
+        sql_params: Optional jinja2 params
+
+    Returns:
+        json
+    """
+    key = request.headers['x-api-key']
+    args = request.json
+    sql = args['sql']
+    sql_params = args.get('sql_params', '{}')
+    rendered_sql = Environment().from_string(
+        sql).render(dict(**literal_eval(sql_params)))
+    autocommit = args.get('autocommit', False)
+    raise_if_not_exists(Backend)
+    backend_results = {item.name: item.credentials
+                       for item in Backend.query('name', name__eq=backend)}
+    credentials = backend_results[backend]
+    backend_client = SqlClient(decrypt(credentials, key=key), autocommit)
+    doer = backend_client.sql_doer()
+    try:
+        results = doer(rendered_sql)
+    except OperationalError as err:
+        return str(err), 400
+    except ProgrammingError as err:
+        return str(err), 400
+    except Exception as err:
+        return str(err), 400
+    return jsonify(str("%s row(s) affected." % results.rowcount))
+
+
+if __name__ == '__main__':
+    DEBUG = False if env['STAGE'] == 'prod' else True
+    api.run(debug=DEBUG, port=5001)

api/docker/bashrc

@@ -0,0 +1,3 @@
+GREEN="\[$(tput setaf 2)\]" 
+RESET="\[$(tput sgr0)\]" 
+export PS1="${GREEN}zappa${RESET}> "

api/docker/yum.conf

@@ -0,0 +1,3 @@
+[main]
+keepcache=0
+releasever=2015.09

api/engine.py

@@ -0,0 +1,37 @@
+"""Engine Builder."""
+from sqlalchemy import create_engine
+
+def _create_engine(engine, autocommit=True):
+    """Return an engine from standard format."""
+    base_con_str = (
+        '{dialect}+{driver}://{username}:{password}@{host}'
+        ':{port}/{database}').format(**engine)
+
+    return create_engine(
+        base_con_str,
+        execution_options=dict(autocommit=autocommit))
+
+
+class SqlClient(object):
+    """SQL Client for any engine in ENGINES.
+
+    >>> redshift = SqlClient('REDSHIFT')
+    >>> viewer = redshift.sql_viewer()
+    >>> viewer('select * from table;')
+    __results__ #  as pd.DataFrame
+    >>> doer = redshift.sql_doer()
+    >>> doer('insert into table things;')
+    1 row affected
+    """
+
+    def __init__(self, engine, autocommit=True):
+        """Just need engine from ENGINES object."""
+        self.engine = _create_engine(engine, autocommit)
+
+    def sql_viewer(self):
+        """Convenience function to view query results."""
+        return self.engine.execute
+
+    def sql_doer(self):
+        """Convenience function to execute table mutations."""
+        return self.engine.execute

api/middleware.py

@@ -0,0 +1,25 @@
+#! /usr/bin/env python2
+import urllib
+
+from flask import url_for
+
+def list_routes(app):
+    output = []
+    for rule in app.url_map.iter_rules():
+
+        if rule.endpoint == 'static':
+            continue
+
+        options = {}
+        for arg in rule.arguments:
+            options[arg] = "[{0}]".format(arg)
+
+        methods = ','.join(rule for rule in rule.methods if rule not in ('OPTIONS','HEAD'))
+        url = url_for(rule.endpoint, **options)
+        line = urllib.unquote("{:7s}{:40s}{}".format(methods, url, rule.endpoint))
+        output.append(line)
+
+    return [line for line in sorted(output)]
+
+
+

api/models.py

@@ -0,0 +1,17 @@
+"""Pynamodb is similar to most ORMs in nature"""
+import boto3
+from pynamodb.models import Model
+from pynamodb.attributes import (
+    UnicodeAttribute, BooleanAttribute)
+
+class Backend(Model):
+    class Meta:
+        # STAGE env always available in Zappa
+        table_name = 'Backend'
+        region = boto3.Session().region_name
+        read_capacity_units = 1
+        write_capacity_units = 1
+
+    name = UnicodeAttribute(hash_key=True)
+    description = UnicodeAttribute()
+    credentials = UnicodeAttribute()

api/requirements.txt

@@ -0,0 +1,9 @@
+zappa
+flask
+python-jose
+pynamodb
+sqlalchemy
+# your database dependencies here
+redshift-sqlalchemy
+psycopg2
+pymssql