implement remote kernel connection through SSH

This uses the --ssh option of the ipython console app to create the necessary
tunnels and create a new connection file with "-ssh" added (because it may use
different ports). Therefore, the REPL is created before the driver. The
connection file naming convention of "emacs-*.ssh" is abandoned in order to
support arbitrary remote kernel connection files. Fixes #11.

Author: smartass101

README.org

@@ -154,6 +154,62 @@
      : (2, [['a', 1, 2], ['b', 2, 3], ['c', 3, 4]])
    #+END_SRC
 
+*** SSH connection to remote kernels
+
+It is possible to connect to a remote kernel running on some server to which SSH
+tunnels can be created. Usually the kernel process is started in a terminal
+multiplexer like =screen= or =tmux= to keep it running after disconnecting. When
+a kernel is started on the remote server using the following command
+
+#+BEGIN_SRC sh
+ipython kernel
+#+END_SRC
+
+The name of the connection file for later use with the =--existing= option of
+=ipython= is printed out. By default has the form =kernel-${PID}.json=, but a
+custom name can be specified with
+
+#+BEGIN_SRC sh
+ipython kernel --IPKernelApp.connection_file=${custom_name}.json
+#+END_SRC
+
+The connection file is written in the runtime directory on the server when using
+Jupyter (IPython >= 4.0) which is usually =/run/user/${UID}/jupyter/= or falls
+back to =${HOME}/.local/share/jupyter/runtime/=. The exact location can be found
+on the server with
+
+#+BEGIN_SRC sh
+python -c "from jupyter_core.paths import jupyter_runtime_dir as p; print(p())"
+#+END_SRC
+
+When using the older IPython < 4.0, the connection file is written in the
+=security/= subdirectory of the current profile, e.g.
+=$HOME/.ipython/profile_default/security/= with the default profile. The profile
+directory can also be found with =ipython locate=.
+
+
+The connection contains connection information (secret hash, ports) and has to
+be transferred from the server to the appropriate directory on the client
+machine, e.g. with the standard runtime directory with Jupyter it could be
+
+#+BEGIN_SRC sh
+scp server:/run/user/\$UID/jupyter/${connnection_file} /run/user/$UID/jupyter 
+#+END_SRC
+
+The backslash before the first =$= makes sure the user id on the server is used
+in the source path.
+
+Finally, the remote session can be connected through specifying both of the
+following arguments in the source blocks:
+- =:session ${connection_file_basename}= :: Will be used to find the
+     connection file, specified without the =.json= extension.
+- =:ssh ${server_name_or_ip}= :: Will be passed to the =--ssh= option of
+     =ipython console= which will establish needed SSH tunnels (nothing more, no
+     shell session is created) and write a modified connection file (that is
+     handled internally). It is best to have SSH configured in such a way that
+     the user does not have to be specified (option =User= in ssh configuration)
+     and possibly password-less login through SSH keys (option =IdentityFile= in
+     ssh configuration).
 ** What features are there outside of Org SRC block evaluation?
 
    * You can ask the running IPython kernel for documentation. Open a
@@ -211,10 +267,28 @@
 
    * Open a REPL using =C-c C-v C-z= so that you get completion in Python buffers.
 
+   * The source block header arguments can be set as a special  property
+
+#+BEGIN_SRC org
+#+PROPERTY: header-args:ipython :session kernel_name :ssh remote_server
+#+END_SRC
+ and then they don't have to be specified for each cell, they are used during
+ execution automatically. However, they are set only after (re)loading the Org
+ file or by =C-c C-c= on this line.
+
 ** Help, it doesn't work
 
    First thing to do is check that you have all of the required
    dependencies. Several common problems have been resolved in the
    project's issues, so take a look there to see if your problem has a
    quick fix. Otherwise feel free to cut an issue - I'll do my best to
    help.
+*** Errors during and/or long session start-up
+This is because it takes a few seconds for the driver server process to bind to
+a port and the REPL to connect to the kernel and possibly establishing SSH
+tunnels. Executing code before the session is fully established can result in
+errors. Because there is no simple way (to the best knowledge of the authors) to
+find out when the session is fully established, the code execution is postponed
+by several seconds. This waiting period can be customized by the
+`ob-ipython-connection-wait` variable in case it is too short or needlessly
+long.

driver.py

@@ -1,8 +1,8 @@
 try:                            # Jupyter and IPython >= 4.0
     import jupyter_client as client
-    find_connection_file = client.find_connection_file
+    client_utils = client
 except ImportError:             # IPython 3
-    from IPython.lib.kernel import find_connection_file
+    import IPython.lib.kernel as client_utils
     import IPython.kernel.blocking.client as client
 
 import sys
@@ -43,7 +43,7 @@ def msg_router(name, ch):
 clients = {}
 
 def create_client(name):
-    cf = find_connection_file('emacs-' + name)
+    cf = client_utils.find_connection_file(name + '.json')
     c = client.BlockingKernelClient(connection_file=cf)
     c.load_connection_file()
     c.start_channels()
@@ -105,8 +105,8 @@ def get(self):
 
 def make_app():
     return tornado.web.Application([
-        tornado.web.url(r"/execute/(\w+)", ExecuteHandler),
-        tornado.web.url(r"/inspect/(\w+)", InspectHandler),
+        tornado.web.url(r"/execute/(\S+)", ExecuteHandler),
+        tornado.web.url(r"/inspect/(\S+)", InspectHandler),
         tornado.web.url(r"/debug", DebugHandler),
         ])
 

ob-ipython.el

@@ -62,6 +62,10 @@
   "Path to the driver script."
   :group 'ob-ipython)
 
+(defcustom ob-ipython-connection-wait 2
+  "Seconds to wait for connections to be established."
+  :group 'ob-ipython)
+
 ;;; utils
 
 (defun ob-ipython--write-base64-string (file b64-string)
@@ -120,11 +124,12 @@
 ;;; process management
 
 (defun ob-ipython--kernel-cmd (name)
-  (-concat (list "ipython" "kernel" (format "--IPKernelApp.connection_file=emacs-%s.json" name))
+  (-concat (list "ipython" "kernel" (format "--IPKernelApp.connection_file=%s.json" name))
            ob-ipython-kernel-extra-args))
 
-(defun ob-ipython--kernel-repl-cmd (name)
-  (list "ipython" "console" "--existing" (format "emacs-%s.json" name)))
+(defun ob-ipython--kernel-repl-cmd (name ssh)
+  (-concat (list "ipython" "console" "--existing" (format "%s.json" name))
+           (if ssh (list "--ssh" ssh))))
 
 (defun ob-ipython--create-process (name cmd)
   (apply 'start-process name (format "*ob-ipython-%s*" name) (car cmd) (cdr cmd)))
@@ -153,14 +158,17 @@
                                       (number-to-string ob-ipython-driver-port)))
     ;; give driver a chance to bind to a port and start serving
     ;; requests. so horrible; so effective.
-    (sleep-for 1)))
+    (sleep-for ob-ipython-connection-wait)))
 
 (defun ob-ipython--get-driver-process ()
   (get-process "ob-ipython-driver"))
 
-(defun ob-ipython--create-repl (name)
-  (run-python (s-join " " (ob-ipython--kernel-repl-cmd name)) nil nil)
-  (format "*%s*" python-shell-buffer-name))
+(defun ob-ipython--create-repl (name ssh)
+  (run-python (s-join " " (ob-ipython--kernel-repl-cmd name ssh)) nil nil)
+  (format "*%s*" python-shell-buffer-name)
+  ;; SSH tunnels take some time to establish and we must wait for the modified
+  ;; connection file to be written for the driver
+  (if ssh (sleep-for ob-ipython-connection-wait)))
 
 ;;; kernel management
 
@@ -297,13 +305,14 @@ a new kernel will be started."
 This function is called by `org-babel-execute-src-block'."
   (let* ((file (cdr (assoc :file params)))
          (session (cdr (assoc :session params)))
+         (ssh (cdr (assoc :ssh params)))
          (result-type (cdr (assoc :result-type params))))
-    (org-babel-ipython-initiate-session session)
+    (org-babel-ipython-initiate-session session params)
     (-when-let (ret (ob-ipython--eval
                      (ob-ipython--execute-request
                       (org-babel-expand-body:generic (encode-coding-string body 'utf-8)
                                                      params (org-babel-variable-assignments:python params))
-                      (ob-ipython--normalize-session session))))
+                      (ob-ipython--normalize-session (if ssh (concat session "-ssh") session)))))
       (let ((result (cdr (assoc :result ret)))
             (output (cdr (assoc :output ret))))
         (if (eq result-type 'output)
@@ -330,9 +339,11 @@ VARS contains resolved variable references"
   (if (string= session "none")
       (error "ob-ipython currently only supports evaluation using a session.
 Make sure your src block has a :session param.")
-    (ob-ipython--create-driver)
-    (ob-ipython--create-kernel (ob-ipython--normalize-session session))
-    (ob-ipython--create-repl (ob-ipython--normalize-session session))))
+    (let ((ssh (cdr (assoc :ssh params)))
+          (nsession (ob-ipython--normalize-session session)))
+      (if (not ssh) (ob-ipython--create-kernel nsession))
+      (ob-ipython--create-repl nsession ssh)
+      (ob-ipython--create-driver))))
 
 (provide 'ob-ipython)