Json upload mode (#21)

* Refactor parameter collection to ignore upload mode and add upload mode retrieval function

* Remove upload mode handling from parameter collection and related functions

* Add base64 download handler and refactor existing download functions

* Add error handling test for PlainDownloadHandler and implement StoreRawForTesting method

* Add tests for PlainDownloadHandler with base64 decoding scenarios

* Enhance base64 decoding in download handler and update HTML documentation for advanced mode

* Update README to include base64 encoding option for data retrieval

* Update README to clarify key pair usage and enhance feature descriptions

README.md

@@ -3,15 +3,17 @@
 
 # iot-ephemeral-value-store
 
-This project provides a simple HTTP server that offers ephemeral storage for IoT data. It generates unique key pairs for data upload and retrieval, stores data temporarily based on a configurable duration, and allows data to be fetched in both JSON and plain text formats.
+This project provides a simple HTTP server that offers ephemeral storage for IoT data. It generates unique key pairs for data upload and retrieval, stores data temporarily based on a configurable duration, and allows data to be fetched in both JSON and plain text formats. The key pairs ensure that you can securely share the download key, but only you can upload data using the upload key.
 
 ## Features
 
 - **Key Pair Generation**: Generate unique upload and download keys for secure data handling.
 - **Data Upload**: Upload data with a simple GET request using the generated upload key.
-- **Data Retrieval**: Retrieve stored data using the download key, either as JSON or plain text for specific data fields.
-- **Patch Feature**: Combine different uploads into a single JSON structure, which can be downloaded with one call.
+- **Data Retrieval**: Retrieve stored data using the download key, either as JSON, plain text, or base64 encoded for specific data fields.
+- **Patch Feature**: Combine different uploads into a single JSON structure, which can be downloaded with one call. Use one upload key for multiple values.
 - **Privacy**: Separate keys for upload and download to ensure secure and private data handling.
+- **Base64 Decoding**: Server-side base64 decoding for easy usage.
+- **Simple HTTP GET**: All operations are done using HTTP GET requests, making it compatible with very simple IoT technology.
 
 ## Why?
 

httphandler/datahandling.go

@@ -5,17 +5,6 @@ import (
 	"time"
 )
 
-func collectParams(params map[string][]string) map[string]string {
-	paramMap := make(map[string]string)
-	for key, values := range params {
-		if len(values) > 0 {
-			sanitizedValue := sanitizeInput(values[0])
-			paramMap[key] = sanitizedValue
-		}
-	}
-	return paramMap
-}
-
 func addTimestampToThisData(paramMap map[string]string, path string) {
 	// if using patch and path is empty than add a timestamp with the value suffix
 	if path == "" {

httphandler/downloadHandler.go

@@ -1,6 +1,7 @@
 package httphandler
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -9,7 +10,11 @@ import (
 	"github.com/gorilla/mux"
 )
 
-func (c Config) PlainDownloadHandler(w http.ResponseWriter, r *http.Request) {
+func (c Config) DownloadPlainHandler(w http.ResponseWriter, r *http.Request) {
+	c.downloadPlainHandler(w, r, false)
+}
+
+func (c Config) downloadPlainHandler(w http.ResponseWriter, r *http.Request, base64mode bool) {
 	vars := mux.Vars(r)
 	downloadKey := vars["downloadKey"]
 	param := vars["param"]
@@ -49,14 +54,46 @@ func (c Config) PlainDownloadHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// If base64 mode is enabled, decode the value from base64url
+	if base64mode {
+		decoded, err := decodeBase64URL(value.(string))
+		if err != nil {
+			c.StatsInstance.IncrementHTTPErrors()
+			http.Error(w, "Error decoding base64url", http.StatusInternalServerError)
+			return
+		}
+		value = decoded
+	}
+
 	c.StatsInstance.IncrementDownloads()
 
 	// Return the value as plain text
 	w.Header().Set("Content-Type", "text/plain")
 	fmt.Fprintln(w, value)
 }
 
-func (c Config) DownloadHandler(w http.ResponseWriter, r *http.Request) {
+func decodeBase64URL(encoded string) (string, error) {
+	// Try decoding with standard base64
+	decodedBytes, err := base64.StdEncoding.DecodeString(encoded)
+	if err == nil {
+		return string(decodedBytes), nil
+	}
+
+	// Try decoding with base64url
+	decodedBytes, err = base64.URLEncoding.DecodeString(encoded)
+	if err == nil {
+		return string(decodedBytes), nil
+	}
+
+	// Try decoding with base64url without padding
+	decodedBytes, err = base64.StdEncoding.WithPadding(base64.NoPadding).DecodeString(encoded)
+	if err != nil {
+		return "", err
+	}
+	return string(decodedBytes), nil
+}
+
+func (c Config) DownloadJsonHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	downloadKey := vars["downloadKey"]
 
@@ -73,3 +110,7 @@ func (c Config) DownloadHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	w.Write(jsonData)
 }
+
+func (c Config) DownloadBase64Handler(w http.ResponseWriter, r *http.Request) {
+	c.downloadPlainHandler(w, r, true)
+}

httphandler/downloadHandler_test.go

@@ -1,6 +1,7 @@
 package httphandler
 
 import (
+	"encoding/base64"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -12,8 +13,9 @@ import (
 
 func Test_PlainDownloadHandler(t *testing.T) {
 	type args struct {
-		w http.ResponseWriter
-		r *http.Request
+		w          http.ResponseWriter
+		r          *http.Request
+		base64mode bool
 	}
 	tests := []struct {
 		name                   string
@@ -100,11 +102,126 @@ func Test_PlainDownloadHandler(t *testing.T) {
 			expectedHTTPErrorCount: 0,
 			expectedDownloadCount:  1,
 		},
+		{
+			name: "PlainDownloadHandler - error decoding JSON",
+			c: Config{
+				StatsInstance: stats.NewStats(),
+				StorageInstance: func() storage.Storage {
+					s := storage.NewInMemoryStorage()
+					// Store invalid JSON data
+					s.StoreRawForTesting("validKey", []byte(`{"key": "value"`)) // Missing closing brace
+					return s
+				}(),
+			},
+			args: args{
+				w: httptest.NewRecorder(),
+				r: func() *http.Request {
+					req, _ := http.NewRequest("GET", "/download/validKey/key", nil)
+					vars := map[string]string{
+						"downloadKey": "validKey",
+						"param":       "key",
+					}
+					return mux.SetURLVars(req, vars)
+				}(),
+			},
+			expectedStatus:         http.StatusInternalServerError,
+			expectedBody:           "Error decoding JSON\n",
+			expectedHTTPErrorCount: 1,
+			expectedDownloadCount:  0,
+		},
+		{
+			name: "PlainDownloadHandler - invalid parameter path",
+			c: Config{
+				StatsInstance: stats.NewStats(),
+				StorageInstance: func() storage.Storage {
+					s := storage.NewInMemoryStorage()
+					data := map[string]interface{}{"key": "value"}
+					s.Store("validKey", data)
+					return s
+				}(),
+			},
+			args: args{
+				w: httptest.NewRecorder(),
+				r: func() *http.Request {
+					req, _ := http.NewRequest("GET", "/download/validKey/key/invalidPath", nil)
+					vars := map[string]string{
+						"downloadKey": "validKey",
+						"param":       "key/invalidPath",
+					}
+					return mux.SetURLVars(req, vars)
+				}(),
+			},
+			expectedStatus:         http.StatusBadRequest,
+			expectedBody:           "Invalid parameter path\n",
+			expectedHTTPErrorCount: 1,
+			expectedDownloadCount:  0,
+		},
+		{
+			name: "PlainDownloadHandler - error decoding base64url",
+			c: Config{
+				StatsInstance: stats.NewStats(),
+				StorageInstance: func() storage.Storage {
+					s := storage.NewInMemoryStorage()
+					data := map[string]interface{}{"key": "invalid_base64"}
+					s.Store("validKey", data)
+					return s
+				}(),
+			},
+			args: args{
+				w: httptest.NewRecorder(),
+				r: func() *http.Request {
+					req, _ := http.NewRequest("GET", "/download/validKey/plain-from-base64url/key", nil)
+					vars := map[string]string{
+						"downloadKey": "validKey",
+						"param":       "key",
+					}
+					return mux.SetURLVars(req, vars)
+				}(),
+				base64mode: true,
+			},
+			expectedStatus:         http.StatusInternalServerError,
+			expectedBody:           "Error decoding base64url\n",
+			expectedHTTPErrorCount: 1,
+			expectedDownloadCount:  0,
+		},
+		{
+			name: "PlainDownloadHandler - decoding base64url",
+			c: Config{
+				StatsInstance: stats.NewStats(),
+				StorageInstance: func() storage.Storage {
+					s := storage.NewInMemoryStorage()
+					base64string := base64.URLEncoding.EncodeToString([]byte("Hallo Welt!"))
+					data := map[string]interface{}{"key": base64string}
+					s.Store("validKey", data)
+					return s
+				}(),
+			},
+			args: args{
+				w: httptest.NewRecorder(),
+				r: func() *http.Request {
+					req, _ := http.NewRequest("GET", "/download/validKey/plain-from-base64url/key", nil)
+					vars := map[string]string{
+						"downloadKey": "validKey",
+						"param":       "key",
+					}
+					return mux.SetURLVars(req, vars)
+				}(),
+				base64mode: true,
+			},
+			expectedStatus:         http.StatusOK,
+			expectedBody:           "Hallo Welt!\n",
+			expectedHTTPErrorCount: 0,
+			expectedDownloadCount:  1,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.c.PlainDownloadHandler(tt.args.w, tt.args.r)
+			if tt.args.base64mode {
+				tt.c.DownloadBase64Handler(tt.args.w, tt.args.r)
+			} else {
+				tt.c.DownloadPlainHandler(tt.args.w, tt.args.r)
+			}
 
 			resp := tt.args.w.(*httptest.ResponseRecorder)
 			if resp.Code != tt.expectedStatus {
@@ -191,7 +308,7 @@ func Test_DownloadHandler(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tt.c.DownloadHandler(tt.args.w, tt.args.r)
+			tt.c.DownloadJsonHandler(tt.args.w, tt.args.r)
 
 			resp := tt.args.w.(*httptest.ResponseRecorder)
 			if resp.Code != tt.expectedStatus {

httphandler/utils.go

@@ -15,3 +15,14 @@ func jsonResponse(w http.ResponseWriter, data interface{}) {
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(data)
 }
+
+func collectParams(params map[string][]string) map[string]string {
+	paramMap := make(map[string]string)
+	for key, values := range params {
+		if len(values) > 0 {
+			sanitizedValue := sanitizeInput(values[0])
+			paramMap[key] = sanitizedValue
+		}
+	}
+	return paramMap
+}

main.go

@@ -125,15 +125,16 @@ func createRouter(hhc httphandler.Config, mc middleware.Config, stats *stats.Sta
 	// Legacy routes
 	r.HandleFunc("/{uploadKey}", hhc.UploadHandler).Methods("GET")
 	r.HandleFunc("/{uploadKey}/", hhc.UploadHandler).Methods("GET")
-	r.HandleFunc("/{downloadKey}/json", hhc.DownloadHandler).Methods("GET")
-	r.HandleFunc("/{downloadKey}/plain/{param}", hhc.PlainDownloadHandler).Methods("GET")
+	r.HandleFunc("/{downloadKey}/json", hhc.DownloadJsonHandler).Methods("GET")
+	r.HandleFunc("/{downloadKey}/plain/{param}", hhc.DownloadPlainHandler).Methods("GET")
 
 	// New routes
 	r.HandleFunc("/u/{uploadKey}", hhc.UploadHandler).Methods("GET")
 	r.HandleFunc("/u/{uploadKey}/", hhc.UploadHandler).Methods("GET")
 
-	r.HandleFunc("/d/{downloadKey}/json", hhc.DownloadHandler).Methods("GET")
-	r.HandleFunc("/d/{downloadKey}/plain/{param:.*}", hhc.PlainDownloadHandler).Methods("GET")
+	r.HandleFunc("/d/{downloadKey}/json", hhc.DownloadJsonHandler).Methods("GET")
+	r.HandleFunc("/d/{downloadKey}/plain/{param:.*}", hhc.DownloadPlainHandler).Methods("GET")
+	r.HandleFunc("/d/{downloadKey}/plain-from-base64url/{param:.*}", hhc.DownloadBase64Handler).Methods("GET")
 	// New routes with nestetd paths, eg. /u/1234/param1
 	r.HandleFunc("/patch/{uploadKey}", hhc.UploadAndPatchHandler).Methods("GET")
 	r.HandleFunc("/patch/{uploadKey}/{param:.*}", hhc.UploadAndPatchHandler).Methods("GET")

main_router_test.go

@@ -111,6 +111,20 @@ func TestRoutesUploadDownload(t *testing.T) {
 	runTests(t, router, tests)
 }
 
+func TestRoutesUploadDownloadWithBase64(t *testing.T) {
+	stats, httphandlerConfig, middlewareConfig := createTestEnvironment(t)
+	router := createRouter(httphandlerConfig, middlewareConfig, stats)
+
+	tests := []testCase{
+		{"Upload", buildURL("/u/%s/?value=SGFsbG8gV2VsdCEK", keyUp), http.StatusOK, true, "Data uploaded successfully", ""},
+		{"Download Plain", buildURL("/d/%s/plain/value", keyDown), http.StatusOK, true, "SGFsbG8gV2VsdCEK\n", ""},
+		{"Download Plain decoded from Base64url", buildURL("/d/%s/plain-from-base64url/value", keyDown), http.StatusOK, true, "Hallo Welt!\n", ""},
+		{"Download JSON", buildURL("/d/%s/json", keyDown), http.StatusOK, true, "\"value\":\"SGFsbG8gV2VsdCEK\"", ""},
+	}
+
+	runTests(t, router, tests)
+}
+
 func TestRoutesUploadDownloadDelete(t *testing.T) {
 	stats, httphandlerConfig, middlewareConfig := createTestEnvironment(t)
 	router := createRouter(httphandlerConfig, middlewareConfig, stats)

static/index.html

@@ -95,6 +95,14 @@ <h4>Delete data (or wait for retention time)</h4>
             <a class="dynamic-link" data-path="/delete/{{.UploadKey}}">/u/{{.UploadKey}}/?name=value</a><br>
         </p>
 
+        <h2>Advanded mode (base64)</h2>
+        <p>
+            <h4>Upload data base64url encoded</h4>
+            <a class="dynamic-link" data-path="/u/{{.UploadKey}}/?some_name=SGFsbG8gV2VsdCE">/u/{{.UploadKey}}/?some_name=SGFsbG8gV2VsdCE</a><br>
+            <h4>Download data base64url decoded</h4>
+            <a class="dynamic-link" data-path="/d/{{.DownloadKey}}/plain-from-base64url/some_name">/d/{{.DownloadKey}}/plain-from-base64url/some_name</a><br>
+        </p>
+
         <h2>Server Settings</h2>
         <ul>
             <li><strong>Software Version:</strong> {{.Version}}</li>

storage/storage.go

@@ -111,3 +111,10 @@ func (c StorageInstance) Delete(downloadKey string) error {
 		return txn.Delete([]byte(downloadKey))
 	})
 }
+
+func (c StorageInstance) StoreRawForTesting(downloadKey string, data []byte) error {
+	return c.Db.Update(func(txn *badger.Txn) error {
+		e := badger.NewEntry([]byte(downloadKey), data).WithTTL(c.PersistDuration)
+		return txn.SetEntry(e)
+	})
+}