added basic testing, minor fixes

.gitignore

@@ -0,0 +1,3 @@
+**/geoip-lookup-service
+**/geoip-lookup
+**/main

README.md

@@ -13,13 +13,6 @@ As I don't have loads of money to spare - Testing is only done using the [IPInfo
 
 If you want to use their extended databases, you might encounter issues.
 
-## State
-
-This project is still in early development.
-
-You are welcome to test it and report issues you find.
-
-
 ## Usage
 
 The binary starts a simple HTTP webserver.
@@ -30,22 +23,59 @@ You can send a query and receive the result as response:
 ```bash
 chmod +x geoip_lookup_service
 ./geoip_lookup_service -l 127.0.0.1 -p 10069 -t ipinfo -country /etc/geoip/country.mmdb -asn /etc/geoip/asn.mmdb -city /etc/geoip/city.mmdb
-# -l = listen address
-# -p = listen port
-# -t = database type (ipinfo/maxmind)
-# -country = path to country-database
-# -city = path to city-database
-# -asn = path to asn-database
+# -l = listen address (default=127.0.0.1)
+# -p = listen port (default=10000)
+# -plain = response in plain text format (default=false)
+# -t = database type (ipinfo/maxmind) (default=ipinfo)
+# -country = path to country-database (default=/etc/geoip/country.mmdb)
+# -city = path to city-database (default=/etc/geoip/city.mmdb)
+# -asn = path to asn-database (default=/etc/geoip/asn.mmdb)
+
+curl "http://127.0.0.1:10069/?lookup=country&ip=1.1.1.1"
+> {"continent":"NA","continent_name":"North America","country":"US","country_name":"United States"}
+
+curl "http://127.0.0.1:10069/?lookup=country&ip=1.1.1.1&filter=country"
+> "US"
+
+curl "http://127.0.0.1:10069/?lookup=asn&ip=1.1.1.1"
+> {"asn":"AS13335","domain":"cloudflare.com","name":"Cloudflare, Inc."}
+
+# use the 'plain' flag to get single attributes without JSON formatting
+./geoip_lookup_service -plain ...
+curl "http://127.0.0.1:10069/?lookup=country&ip=1.1.1.1&filter=country_name"
+> United States
+
+# use other DB-type
+./geoip_lookup_service -t maxmind ...
 
-curl http://127.0.0.1:10069/?lookup=country&ip=1.1.1.1
-> ...
+curl "http://127.0.0.1:10069/?ip=1.1.1.1&lookup=asn"
+> {"autonomous_system_number":13335,"autonomous_system_organization":"CLOUDFLARENET"}
 
-curl http://127.0.0.1:10069/?lookup=country&ip=1.1.1.1&filter=xxx
-> ...
+curl "http://127.0.0.1:10069/?ip=1.1.1.1&lookup=country"
+> {"registered_country":{"geoname_id":2077456,"iso_code":"AU","names":{"de":"Australien","en":"Australia","es":"Australia","fr":"Australie","ja":"オーストラリア","pt-BR":"Austrália","ru":"Австралия","zh-CN":"澳大利亚"}}}
 
-curl http://127.0.0.1:10069/?lookup=asn&ip=1.1.1.1
-> ...
+# filters can also be used to get deeper attributes
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=country"
+> {"continent":{"code":"NA","geoname_id":6255149,"names":{"de":"Nordamerika","en":"North America","es":"Norteamérica","fr":"Amérique du Nord","ja":"北アメリカ","pt-BR":"América do Norte","ru":"Северная Америка","zh-CN":"北美洲"}},"country":{"geoname_id":6252001,"iso_code":"US","names":{"de":"Vereinigte Staaten","en":"United States","es":"Estados Unidos","fr":"États Unis","ja":"アメリカ","pt-BR":"EUA","ru":"США","zh-CN":"美国"}},"registered_country":{"geoname_id":6252001,"iso_code":"US","names":{"de":"Vereinigte Staaten","en":"United States","es":"Estados Unidos","fr":"États Unis","ja":"アメリカ","pt-BR":"EUA","ru":"США","zh-CN":"美国"}}}
 
-curl http://127.0.0.1:10069/?lookup=city&ip=1.1.1.1
-> ...
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=country&filter=country.iso_code"
+> "US"
+
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=country&filter=country.names.en"
+> "United States"
+
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=city&filter=location"
+> {"accuracy_radius":1000,"latitude":37.751,"longitude":-97.822,"time_zone":"America/Chicago"}
 ```
+
+----
+
+## Testing
+
+Basic integration tests are done by using the test-script:
+
+```bash
+bash scripts/test.sh
+```
+
+Feel free to contribute more test-cases if you found some edge-case issue(s).

cnf/main.go

@@ -1,22 +1,3 @@
 package cnf
 
-const PORT = 7069
-
-var LOOKUP_MAPPING = map[uint8]interface{}{
-	DB_TYPE_IPINFO: map[string]interface{}{
-		"country_asn": IPINFO_COUNTRY_ASN,
-		"country":     IPINFO_COUNTRY,
-		"city":        IPINFO_CITY,
-		"asn":         IPINFO_ASN,
-		"privacy":     IPINFO_PRIVACY,
-	},
-	DB_TYPE_MAXMIND: map[string]interface{}{
-		"country_asn": nil,
-		"country":     MAXMIND_COUNTRY,
-		"city":        MAXMIND_CITY,
-		"asn":         MAXMIND_ASN,
-		"privacy":     MAXMIND_PRIVACY,
-	},
-}
-
-var LOOKUP = LOOKUP_MAPPING[DB_TYPE].(map[string]interface{})
+const VERSION = 0.1

cnf/user.go

@@ -5,3 +5,4 @@ var DB_COUNTRY = "/etc/geoip/country.mmdb"
 var DB_CITY = "/etc/geoip/city.mmdb"
 var DB_ASN = "/etc/geoip/asn.mmdb"
 var DB_PRIVACY = "/etc/geoip/privacy.mmdb" // vpns, tor, proxies, hosting
+var RETURN_PLAIN = false

lookup/ipinfo.go

@@ -6,22 +6,22 @@ import (
 	"github.com/superstes/geoip-lookup-service/cnf"
 )
 
-func IpInfoCountry(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_COUNTRY)
+func IpInfoCountry(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.IPINFO_COUNTRY, cnf.DB_COUNTRY)
 }
 
-func IpInfoCity(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_CITY)
+func IpInfoCity(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.IPINFO_CITY, cnf.DB_CITY)
 }
 
-func IpInfoAsn(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_ASN)
+func IpInfoAsn(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.IPINFO_ASN, cnf.DB_ASN)
 }
 
-func IpInfoCountryAsn(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_COUNTRY)
+func IpInfoCountryAsn(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.IPINFO_COUNTRY_ASN, cnf.DB_COUNTRY)
 }
 
-func IpInfoPrivacy(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_PRIVACY)
+func IpInfoPrivacy(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.IPINFO_PRIVACY, cnf.DB_PRIVACY)
 }

lookup/maxmind.go

@@ -6,18 +6,18 @@ import (
 	"github.com/superstes/geoip-lookup-service/cnf"
 )
 
-func MaxMindCountry(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_COUNTRY)
+func MaxMindCountry(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.MAXMIND_COUNTRY, cnf.DB_COUNTRY)
 }
 
-func MaxMindCity(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_CITY)
+func MaxMindCity(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.MAXMIND_CITY, cnf.DB_CITY)
 }
 
-func MaxMindAsn(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_ASN)
+func MaxMindAsn(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.MAXMIND_ASN, cnf.DB_ASN)
 }
 
-func MaxMindPrivacy(ip net.IP, dataStructure interface{}) (interface{}, error) {
-	return lookupBase(ip, dataStructure, cnf.DB_PRIVACY)
+func MaxMindPrivacy(ip net.IP) (interface{}, error) {
+	return lookupBase(ip, cnf.MAXMIND_PRIVACY, cnf.DB_PRIVACY)
 }

main/main.go

@@ -14,6 +14,7 @@ func welcome() {
 	fmt.Println("/ /_/ /  __/ /_/ // // ____/  / /___/ /_/ / /_/ / ,< / /_/ / /_/ /")
 	fmt.Println("\\____/\\___/\\____/___/_/      /_____/\\____/\\____/_/|_|\\__,_/ .___/ ")
 	fmt.Println("                                                         /_/      ")
+	fmt.Printf("Version: %v\n", cnf.VERSION)
 	fmt.Printf("by Superstes (GPLv3)\n\n")
 }
 
@@ -28,7 +29,8 @@ func main() {
 	flag.StringVar(&cnf.DB_COUNTRY, "country", cnf.DB_COUNTRY, "Path to the country-database (optional)")
 	flag.StringVar(&cnf.DB_CITY, "city", cnf.DB_CITY, "Path to the city-database (optional)")
 	flag.StringVar(&cnf.DB_ASN, "asn", cnf.DB_ASN, "Path to the asn-database (optional)")
-	flag.StringVar(&cnf.DB_PRIVACY, "asn", cnf.DB_PRIVACY, "Path to the privacy-database (optional)")
+	flag.StringVar(&cnf.DB_PRIVACY, "privacy", cnf.DB_PRIVACY, "Path to the privacy-database (optional)")
+	flag.BoolVar(&cnf.RETURN_PLAIN, "plain", cnf.RETURN_PLAIN, "If the result should be returned in plain text format")
 	flag.Parse()
 
 	if dbType == "maxmind" {

main/server.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"strings"
 
 	"github.com/superstes/geoip-lookup-service/cnf"
 	"github.com/superstes/geoip-lookup-service/lookup"
@@ -15,12 +16,29 @@ import (
 
 func errorResponse(w http.ResponseWriter, m string) {
 	w.WriteHeader(http.StatusBadRequest)
-	_, err := io.WriteString(w, m)
+	_, err := io.WriteString(w, fmt.Sprintf("%v\n", m))
 	if err != nil {
 		log.Fatal(err)
 	}
 }
 
+func returnResult(w http.ResponseWriter, data interface{}) {
+	if cnf.RETURN_PLAIN {
+		w.Header().Set("Content-Type", "text/plain")
+		_, err := io.WriteString(w, fmt.Sprintf("%+v\n", data))
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		w.Header().Set("Content-Type", "application/json")
+		err := json.NewEncoder(w).Encode(data)
+		if err != nil {
+			log.Fatal(err)
+			errorResponse(w, "Failed to JSON-encode data!")
+		}
+	}
+}
+
 func geoIpLookup(w http.ResponseWriter, r *http.Request) {
 	ipStr := r.URL.Query().Get("ip")
 	lookupStr := r.URL.Query().Get("lookup")
@@ -32,51 +50,41 @@ func geoIpLookup(w http.ResponseWriter, r *http.Request) {
 
 	ip := net.ParseIP(ipStr)
 	if ip == nil {
-		errorResponse(w, "Provided IP is not valid!")
+		errorResponse(w, "Invalid IP provided")
 		return
 	}
 
-	dataStructure, lookupExists := cnf.LOOKUP[lookupStr]
-	if !lookupExists || dataStructure == nil {
-		errorResponse(w, "Provided LOOKUP is not valid!")
+	data, err := lookup.FUNC[lookupStr].(func(net.IP) (interface{}, error))(ip)
+	if data == nil {
+		errorResponse(w, "Invalid LOOKUP provided")
 		return
 	}
-
-	data, err := lookup.FUNC[lookupStr].(func(net.IP, interface{}) (interface{}, error))(
-		ip, dataStructure,
-	)
 	if err != nil {
 		log.Fatal(err)
-		errorResponse(w, "Failed to lookup data!")
+		errorResponse(w, "Failed to lookup data")
 		return
 	}
 
 	if filterStr != "" {
-		// todo: allow deeper filtering
-		defer func() {
-			if err := recover(); err != nil {
-				log.Fatal(err)
-				errorResponse(w, "Provided FILTER is not valid!")
+		filteredData := data
+		for _, subFilterStr := range strings.Split(filterStr, ".") {
+			defer func() {
+				if err := recover(); err != nil {
+					log.Fatal(err)
+					errorResponse(w, "Invalid FILTER provided")
+				}
+			}()
+			filteredData = u.GetMapValue(filteredData, subFilterStr)
+			if filteredData == nil {
+				errorResponse(w, "Invalid FILTER provided")
+				return
 			}
-		}()
-		filteredData := u.GetAttribute(data, filterStr)
-		if !filteredData.IsValid() {
-			errorResponse(w, "Provided FILTER is not valid!")
-		}
-		w.Header().Set("Content-Type", "application/json")
-		err := json.NewEncoder(w).Encode(filteredData.String())
-		if err != nil {
-			log.Fatal(err)
-			errorResponse(w, "Failed to JSON-encode data!")
 		}
+		returnResult(w, filteredData)
 		return
 	}
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(data)
-	if err != nil {
-		log.Fatal(err)
-		errorResponse(w, "Failed to JSON-encode data!")
-	}
+
+	returnResult(w, data)
 	return
 }
 

script/build_run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eo pipefail
+
+PORT=10069
+
+DB_TYPE='ipinfo'
+if [ -n "$1" ]
+then
+  DB_TYPE="$1"
+fi
+
+DB_DIR='/etc/geoip'
+if [ -n "$2" ]
+then
+  DB_DIR="$2"
+fi
+
+set -u
+
+DB_CO="${DB_DIR}/${DB_TYPE}_country.mmdb"
+DB_AS="${DB_DIR}/${DB_TYPE}_asn.mmdb"
+DB_CI="${DB_DIR}/${DB_TYPE}_city.mmdb"
+
+if ! [ -f "$DB_CO" ] || ! [ -f "$DB_AS" ] || ([[ "$DB_TYPE" == "maxmind" ]] && ! [ -f "$DB_CI" ])
+then
+  echo "ERROR: Required databases missing (${DB_DIR}/${DB_TYPE}_*)"
+  exit 1
+fi
+
+
+
+binary="/tmp/geoip_lookup_$(date +"%s")"
+
+cd "$(dirname "$0")/../main"
+go build -o "$binary"
+chmod +x "$binary"
+
+echo "RUNNING GeoIP Lookup: ${DB_DIR}/${DB_TYPE}_*"
+"$binary" -t "$DB_TYPE" -p "$PORT" -country "$DB_CO" -asn "$DB_AS" -city "$DB_CI"

script/test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+echo ''
+echo "### TESTING IPINFO DATBASES ###"
+
+./build_run.sh "ipinfo"  > /dev/null &
+sleep 1
+export DB_TYPE="IPINFO"
+./test_requests_base.sh
+./test_requests_ipinfo.sh
+
+pkill -f "/tmp/geoip_lookup_*" > /dev/null
+sleep 1
+
+echo ''
+echo "### TESTING MAXMIND DATBASES ###"
+
+./build_run.sh "maxmind" > /dev/null &
+sleep 1
+export DB_TYPE="MAXMIND"
+./test_requests_base.sh
+./test_requests_maxmind.sh
+
+pkill -f "/tmp/geoip_lookup_*" > /dev/null
+sleep 1
+
+echo ''
+echo 'FINISHED'
+echo ''

script/test_requests_base.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+
+echo "${DB_TYPE}: Testing COUNTRY"
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=country"
+sleep 1
+
+echo "${DB_TYPE}: Testing ASN"
+curl "http://127.0.0.1:10069/?ip=8.8.8.8&lookup=asn"
+sleep 1