Database and api (#1)

* Adds intial support for database

* Adds option to enable database and api.

* Update Go version in go.mod

* Revise proxy server start commands in README

Updated commands to start the proxy server.

* Refactor port flag parsing in main function

Moved port flag parsing to the beginning of the main function.

---------

Co-authored-by: Holger Wolff <waringer@users.noreply.github.com>

Author: SebbeJohansson

.gitignore

@@ -3,3 +3,9 @@
 *.code-workspace
 **/bin/
 **/obj/
+
+# App specific ignores
+emoProxy.conf
+data/
+# tmp is used for air
+tmp/

README.md

@@ -2,7 +2,10 @@
 This part of the knowledge bank is a Proxy server to analyze the traffic that goes between the EMO robot and the living.ai servers.
 
 ## Start the proxy
-`go run emoProxy.go`
+`go run .`
+
+### Alternatively start with air
+`air run .`
 
 ## Docker setup
 You can find a simplified setup using Docker Compose by SebbeJohansson here: https://github.com/SebbeJohansson/emo-proxy-docker

database.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"database/sql"
+	"log"
+
+	_ "modernc.org/sqlite"
+)
+
+var DB *sql.DB // Capitalized to be visible (though not strictly necessary if in same package)
+
+func InitDB(path string) error {
+	_db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return err
+	}
+	DB = _db // Assign to global DB variable
+	// Create a simple table for intercepted data
+	query := `
+	    CREATE TABLE IF NOT EXISTS requests (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+		endpoint TEXT,
+		payload TEXT,
+		response TEXT
+	    );`
+
+	_, err = DB.Exec(query)
+	return err
+}
+
+func saveRequest(requestEndPoint string, payload string, response string) {
+	log.Println("Saving request to DB...")
+	_, err := DB.Exec("INSERT INTO requests (endpoint, payload, response) VALUES (?, ?, ?)", requestEndPoint, payload, response)
+	if err != nil {
+		log.Println("Failed to save to DB: ", err)
+	}
+}
+
+func getAllRequests() ([]map[string]interface{}, error) {
+	rows, err := DB.Query("SELECT id, timestamp, endpoint, payload, response FROM requests")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var results []map[string]interface{}
+	for rows.Next() {
+		var id int
+		var timestamp string
+		var endpoint string
+		var payload string
+		var response string
+
+		err := rows.Scan(&id, &timestamp, &endpoint, &payload, &response)
+		if err != nil {
+			return nil, err
+		}
+
+		record := map[string]interface{}{
+			"id":        id,
+			"timestamp": timestamp,
+			"endpoint":  endpoint,
+			"payload":   payload,
+			"response":  response,
+		}
+		results = append(results, record)
+	}
+	return results, rows.Err()
+}

emoProxy.conf emoProxy.conf.exampleemoProxy.conf renamed to emoProxy.conf.example

@@ -5,5 +5,7 @@
     "livingio_tts_server": "eu-tts.living.ai",
     "livingio_res_server": "res.living.ai",
     "postFS": "/tmp/",
-    "logFileName": "/var/log/emoProxyss.log"
+    "logFileName": "/var/log/emoProxyss.log",
+    "enableDatabaseAndAPI": false, # For now, default behavior is still without db and api.
+    "sqliteLocation": "/var/data/emo_logs.db"
 }

emoProxy.go

@@ -32,40 +32,63 @@ type Configuration struct {
 	Livingio_RES_Server     string `json:"livingio_res_server"`
 	PostFS                  string `json:"postFS"`
 	LogFileName             string `json:"logFileName"`
+	EnableDatabaseAndAPI    bool   `json:"enableDatabaseAndAPI"`
+	SqliteLocation          string `json:"sqliteLocation"`
 }
 
 var (
-	conf Configuration
+	conf              Configuration
+	useDatabaseAndAPI bool = false
 )
 
 func main() {
+	log.Println("Starting application...")
 	//load config
 	confFile := flag.String("c", "emoProxy.conf", "config file to use")
+	Port := flag.Int("port", 8080, "http port")
 	flag.Parse()
 
 	err := loadConfig(*confFile)
 	if err != nil {
 		log.Println("can't read conf file", *confFile, "- using default config")
 	}
-
+	log.Println("config loaded")
 	writePid()
 
 	// disable ssl checks
 	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 
+	// parse flags
+	log.Println("Starting app on port: ", *Port)
+
 	// redirect log
 	logFile, err := os.OpenFile(conf.LogFileName, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644)
 	if err != nil {
 		log.Panic(err)
 	}
+
 	defer logFile.Close()
 	log.SetOutput(logFile)
 	log.SetFlags(log.Lshortfile | log.LstdFlags)
 
-	// parse flags
-	Port := flag.Int("port", 8081, "http port")
-	flag.Parse()
-	log.Println("Port: ", *Port)
+	useDatabaseAndAPI = conf.EnableDatabaseAndAPI
+
+	if useDatabaseAndAPI {
+		log.Println("Database and API enabled")
+
+		dbPath := conf.SqliteLocation
+		flagDbPath := flag.String("db", "", "path to the sqlite database file")
+		if *flagDbPath != "" {
+			dbPath = *flagDbPath
+		}
+		flag.Parse()
+		dbCreateErr := InitDB(dbPath)
+		if dbCreateErr != nil {
+			log.Panic(dbCreateErr)
+		}
+	} else {
+		log.Println("Note: Database and API disabled")
+	}
 
 	// handle time requests
 	http.HandleFunc("/time", func(w http.ResponseWriter, r *http.Request) {
@@ -152,7 +175,40 @@ func main() {
 		fmt.Fprint(w, body)
 	})
 
-	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*Port), nil))
+	if useDatabaseAndAPI {
+		// proxy-api endpoints
+		http.HandleFunc("/proxy-api/requests", func(w http.ResponseWriter, r *http.Request) {
+			logRequest(r)
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.WriteHeader(http.StatusOK)
+
+			requests, err := getAllRequests()
+			if err != nil {
+				http.Error(w, fmt.Sprintf(`{"error":"%v"}`, err), http.StatusInternalServerError)
+				return
+			}
+			json.NewEncoder(w).Encode(requests)
+		})
+	}
+
+	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*Port), corsMiddleware(http.DefaultServeMux)))
+}
+
+func corsMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Replace "*" with "http://localhost:3000" for better security
+		w.Header().Set("Access-Control-Allow-Origin", "http://localhost:3000")
+		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
+		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization")
+
+		// Handle the preflight OPTIONS request
+		if r.Method == "OPTIONS" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		next.ServeHTTP(w, r)
+	})
 }
 
 func loadConfig(filename string) error {
@@ -164,6 +220,8 @@ func loadConfig(filename string) error {
 		Livingio_RES_Server:     "res.living.ai",
 		PostFS:                  "/tmp/",
 		LogFileName:             "/var/log/emoProxy.log",
+		EnableDatabaseAndAPI:    false,
+		SqliteLocation:          "/var/data/emo_logs.db",
 	}
 
 	bytes, err := os.ReadFile(filename)
@@ -228,16 +286,17 @@ func logBody(contentType string, body []byte, prefix string) {
 
 func makeApiRequest(r *http.Request) string {
 	var request *http.Request
+	var requestBody []byte
 	switch r.Method {
 	case "GET":
 		request, _ = http.NewRequest("GET", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), nil)
 	case "POST":
-		body, _ := io.ReadAll(r.Body)
+		requestBody, _ := io.ReadAll(r.Body)
 
 		// write post request body to fs
-		logBody(r.Header.Get("Content-Type"), body, "apiReq_")
+		logBody(r.Header.Get("Content-Type"), requestBody, "apiReq_")
 
-		request, _ = http.NewRequest("POST", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), bytes.NewBuffer(body))
+		request, _ = http.NewRequest("POST", "https://"+conf.Livingio_API_Server+r.URL.RequestURI(), bytes.NewBuffer(requestBody))
 
 		request.Header.Add("Content-Type", r.Header.Get("Content-Type"))
 		request.Header.Add("Content-Length", r.Header.Get("Content-Length"))
@@ -269,6 +328,10 @@ func makeApiRequest(r *http.Request) string {
 	log.Println("Server response: ", string(body))
 
 	logResponse(response)
+
+	if useDatabaseAndAPI {
+		saveRequest(r.URL.RequestURI(), string(requestBody), string(body))
+	}
 	return string(body)
 }
 
@@ -301,6 +364,10 @@ func makeTtsRequest(r *http.Request) string {
 	// write post request body to fs
 	logBody(response.Header.Get("Content-Type"), body, "tts_")
 	logResponse(response)
+
+	if useDatabaseAndAPI {
+		saveRequest(r.URL.RequestURI(), "", "")
+	}
 	return string(body)
 }
 
@@ -333,6 +400,10 @@ func makeApiTtsRequest(r *http.Request) string {
 	// write post request body to fs
 	logBody(response.Header.Get("Content-Type"), body, "apitts_")
 	logResponse(response)
+
+	if useDatabaseAndAPI {
+		saveRequest(r.URL.RequestURI(), "", string(body))
+	}
 	return string(body)
 }
 
@@ -369,5 +440,9 @@ func makeResRequest(r *http.Request, w http.ResponseWriter) string {
 	}
 
 	logResponse(response)
+
+	if useDatabaseAndAPI {
+		saveRequest(r.URL.RequestURI(), "", string(body))
+	}
 	return string(body)
 }

go.mod

@@ -1,3 +1,18 @@
 module emoProxy
 
-go 1.17
+go 1.24
+
+require modernc.org/sqlite v1.42.2
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
+	golang.org/x/sys v0.36.0 // indirect
+	modernc.org/libc v1.66.10 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

go.sum

@@ -0,0 +1,23 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A=
+modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74=
+modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=