diff --git a/.version b/.version index 63700cc..e53d5aa 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -VERSION=0.2.0 \ No newline at end of file +VERSION=3 \ No newline at end of file diff --git a/README.md b/README.md index 939f89e..aae519a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ tasks: On linux, just run: ```console ╭─adam@box ~/ -╰─➤ sudo curl -s -L https://github.com/Sharpz7/sharpcd/releases/download/0.2.0/install.sh | sudo bash +╰─➤ sudo curl -s -L https://github.com/Sharpz7/sharpcd/releases/download/3/install.sh | sudo bash ``` ## Command Options diff --git a/installs/install.sh b/installs/install.sh index 04efffa..8ef04ee 100644 --- a/installs/install.sh +++ b/installs/install.sh @@ -1,5 +1,18 @@ #/bin/bash +if [[ $1 == "client" ]]; +then + # Download and unpack + wget https://github.com/Sharpz7/sharpcd/releases/download/XXXXX/linux.tar.gz + sudo mkdir -p /tmp/sharpcd + sudo tar -C /tmp/sharpcd -zxvf linux.tar.gz + sudo cp /tmp/sharpcd/sharpcd /usr/local/bin/sharpcd + rm -r linux.tar.gz + + sharpcd --help + exit 0 +fi + echo "Installing required modules" sudo apt-get install -y lsof @@ -10,14 +23,11 @@ sudo kill $(sudo lsof -t -i:5666) > /dev/null 2>&1 || true ver=$(echo $(sharpcd version) | sed "s/^.*Version: \([0-9.]*\).*/\1/") vernum=$(echo "$ver" | sed -r 's/[.0]+//g') -if [[ $vernum =~ ^[0-9]+$ ]]; -then - if [[ $vernum < 0 ]]; - then +if [[ $vernum =~ ^[0-9]+$ ]]; then + if [[ $vernum < 0 ]]; then echo "Breaking changes: Removing old sharpcd-data" read -r -p "Are you sure? [y/N] " response - if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]] - then + if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo "Deleting old Data..." sudo rm -r /usr/local/bin/sharpcd-data else @@ -58,7 +68,6 @@ sudo chmod +x /usr/local/bin/sharpcd sudo chmod 755 /usr/local/bin/sharpcd sudo chown -R sharpcd:sharpcd /usr/local/bin/sharpcd-data - # Create system service test=$(cat <<-END [Unit] @@ -98,4 +107,4 @@ echo "Use the follow IP table commands to block port 5666 from outside localhost echo "sudo iptables -D INPUT -p tcp --dport 5666 -s localhost -j ACCEPT" echo "sudo iptables -D INPUT -p tcp --dport 5666 -j DROP" echo "sudo iptables -A INPUT -p tcp --dport 5666 -s localhost -j ACCEPT" -echo "sudo iptables -A INPUT -p tcp --dport 5666 -j DROP" +echo "sudo iptables -A INPUT -p tcp --dport 5666 -j DROP" \ No newline at end of file diff --git a/sharpdev.yml b/sharpdev.yml index 02f2e6b..6057288 100644 --- a/sharpdev.yml +++ b/sharpdev.yml @@ -9,27 +9,46 @@ scripts: build: | go build -o ./internal/sharpcd ./src - sharpcd1: | + sharpcd: | sharpdev build - ./internal/sharpcd $_ARG1 + ./internal/sharpcd $_ARG1 $_ARG2 $_ARG3 $_ARG4 $_ARG5 - server: | + sharedserver: | sudo apt-get install -y lsof sudo kill $(sudo lsof -t -i:5666) > /dev/null 2>&1 || true sharpdev build - $(sudo ./internal/sharpcd server &) & + + server: | + sharpdev sharedserver + $(sudo ./internal/sharpcd server) & + + serveropen: | + sharpdev sharedserver + sudo ./internal/sharpcd server + + test: | + sharpdev server + sharpdev clientr + sharpdev kill kill: | sudo apt-get install -y lsof sudo kill $(sudo lsof -t -i:5666) > /dev/null 2>&1 || true - client: | + shared: | sudo docker-compose -f "./internal/sharpcd-data/docker/external_task/docker-compose.yml" down $(sudo docker volume rm memes) > /dev/null 2>&1 || true $(sudo docker network rm memes) > /dev/null 2>&1 || true sharpdev build + + client: | + sharpdev shared ./internal/sharpcd --secret $PASSD + clientr: | + sharpdev shared + sharpdev sharpcd --secret $PASSD --remotefile https://raw.githubusercontent.com/Sharpz7/sharpcd/master/sharpcd.yml + trak: | sharpdev build ./internal/sharpcd --secret $PASSD trak $_ARG1 $_ARG2 $_ARG3 @@ -43,16 +62,10 @@ scripts: traklist: | sharpdev trak list local - filter: ./internal/sharpcd addfilter https://raw.githubusercontent.com/Sharpz7/ remove: ./internal/sharpcd removefilter https://raw.githubusercontent.com/Sharpz7/ token: ./internal/sharpcd changetoken $_ARG1 - test: | - git checkout dev - git push origin dev - sharpdev client - alljobsd: curl -k -X POST -d SECRETD https://localhost:5666/api/jobs alljobs: curl -k -X POST -d SECRET https://173.212.252.82:5666/api/jobs logsfeed: curl -k -X POST -d SECRETD https://localhost:5666/api/logsfeed/$_ARG1 diff --git a/src/api.go b/src/api.go index 823ff32..5c08c88 100644 --- a/src/api.go +++ b/src/api.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "os/exec" + "sort" "strings" ) @@ -17,7 +18,13 @@ func getAPIData(r *http.Request, resp *response) error { path := strings.Split(r.URL.Path[5:], "/") switch path[0] { case "jobs": - resp.Jobs = allJobs.List + for _, job := range allJobs { + resp.Jobs = append(resp.Jobs, job) + } + + sort.Slice(resp.Jobs, func(i, j int) bool { + return resp.Jobs[i].ID < resp.Jobs[j].ID + }) return nil case "job": resp.Job, err = getJobs(path[1]) @@ -65,8 +72,8 @@ func getLogsFeed(path string) (string, error) { func getJobs(path string) (*taskJob, error) { var emptyJob *taskJob - for _, job := range allJobs.List { - if job.ID == path { + for id, job := range allJobs { + if id == path { err := checkJobStatus(job) return job, err } @@ -77,8 +84,10 @@ func getJobs(path string) (*taskJob, error) { func checkJobStatus(job *taskJob) error { logs, err := getLogs(job.ID) - if strings.Contains(logs, "exited with code") { - job.Status = jobStatus.Stopped + + exited := strings.Contains(logs, "exited with code") + if exited && (job.Status != jobStatus.Building) { + job.Status = jobStatus.Errored job.ErrMsg = "A Container has maybe exited unexpectedly" } diff --git a/src/client.go b/src/client.go index 379e685..87fe1b6 100644 --- a/src/client.go +++ b/src/client.go @@ -20,12 +20,27 @@ import ( func client() { var globalErr bool - - // Get config, get data from it - f, err := ioutil.ReadFile("./sharpcd.yml") var con config - err = yaml.Unmarshal(f, &con) - handle(err, "Failed to read and extract sharpcd.yml") + var err error + var file []byte + + if len(secretFlag) == 0 { + resp, err := http.Get(secretFlag) + handle(err, "Failed to download remote sharpcd.yml") + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + file, err = ioutil.ReadAll(resp.Body) + handle(err, "Failed to read remote sharpcd.yml") + } + } else { + // Get config, get data from it + file, err = ioutil.ReadFile("./sharpcd.yml") + handle(err, "Failed to read and extract sharpcd.yml") + } + + err = yaml.Unmarshal(file, &con) + handle(err, "Failed to read yaml from sharpcd.yml") // POST to sharpcd server for each task for id, task := range con.Tasks { @@ -183,7 +198,7 @@ func postCommChecks(t task, id string) error { } if lastIssue != job.Issue { - fmt.Println("No fatal Issue found: " + job.Issue) + fmt.Println("Non fatal Issue found: " + job.Issue) lastIssue = job.Issue } diff --git a/src/jobs.go b/src/jobs.go index 47eefbe..c6ddefd 100644 --- a/src/jobs.go +++ b/src/jobs.go @@ -2,27 +2,26 @@ package main import ( "errors" + "fmt" "io/ioutil" "net/http" "os" "os/exec" "regexp" "strings" + "time" "github.com/joho/godotenv" ) -// Pointer to variable storing all jobs -var allJobs = &allTaskJobs{} - func getJob(id string) *taskJob { // If there are no jobs to chose from - if len(allJobs.List) == 0 { + if len(allJobs) == 0 { return nil } // Return a job with the matching ID - for _, job := range allJobs.List { + for _, job := range allJobs { if job.ID == id { return job } @@ -40,21 +39,25 @@ func createJob(payload postData) { URL: payload.GitURL + payload.Compose, Enviroment: payload.Enviroment, Registry: payload.Registry, - Reconnect: false} + ErrMsg: "", + Reconnect: false, + Kill: false} // If a job with that ID already exists if job := getJob(payload.ID); job != nil { - // Assign its ID to the old job ID + // Kill the old jobs logging + job.Kill = true + newJob.ID = job.ID + allJobs[payload.ID] = &newJob - // Replace and run the new job - *job = newJob - job.Run() + newJob.Run() } else { newJob.ID = payload.ID - allJobs.List = append(allJobs.List, &newJob) + + allJobs[newJob.ID] = &newJob comm := &newJob comm.Run() } @@ -65,31 +68,68 @@ func (job *taskJob) Run() { var cmd *exec.Cmd + // All Location Data + id := job.ID + logsLoc := folder.Docker + id + composeLoc := folder.Docker + id + "/docker-compose.yml" + // Mark as building job.Status = jobStatus.Building // Run the correct job Type switch job.Type { case "docker": - cmd = job.DockerCmd() + + // Setup the job + if job.Reconnect != true { + job.DockerSetup() + } + + // Get logging Running + cmd = exec.Command("docker-compose", "-f", composeLoc, "up", "--no-color") + cmd.Env = job.insertEnviroment() } // If setting up the command went fine if job.Status != jobStatus.Errored { + // Empty the logs file + outfile, err := os.Create(logsLoc + "/info.log") + handleAPI(err, job, "Failed to create log file") + job.ErrMsg = "" + cmd.Stdout = outfile + // Run Command job.Status = jobStatus.Running - err := cmd.Run() + err = cmd.Run() handleAPI(err, job, "Job Exited With Error") job.Status = jobStatus.Stopped + fmt.Println("Stopped:", job.ID, "Kill:", job.Kill) + + go job.DockerLog(cmd) + + } +} + +// Log the dockerfile until the job has been killed +func (job *taskJob) DockerLog(cmd *exec.Cmd) { + + fmt.Println("Running extended logs") + + for job.Kill == false { + cmd.Run() + + time.Sleep(1 * time.Second) } + + fmt.Println("Exited:", job.ID) } // Get cmd for a Docker Job -func (job *taskJob) DockerCmd() *exec.Cmd { +func (job *taskJob) DockerSetup() { // All Location Data id := job.ID @@ -97,97 +137,81 @@ func (job *taskJob) DockerCmd() *exec.Cmd { logsLoc := folder.Docker + id composeLoc := folder.Docker + id + "/docker-compose.yml" - var err error + // Get github token + f, err := readFilter() + if err != nil { + handleAPI(err, job, "Failed to get Token") + } - if job.Reconnect != true { - // Get github token - f, err := readFilter() - if err != nil { - handleAPI(err, job, "Failed to get Token") - } + // Make url, read the compose file + req, err := http.NewRequest("GET", url, nil) + if err != nil { + handleAPI(err, job, "Failed to build request") + } - // Make url, read the compose file - req, err := http.NewRequest("GET", url, nil) - if err != nil { - handleAPI(err, job, "Failed to build request") - } + if f.Token != "" { + req.Header.Set("Authorization", "token "+f.Token) + req.Header.Set("Accept", "application/vnd.github.v3.raw") + } - if f.Token != "" { - req.Header.Set("Authorization", "token "+f.Token) - req.Header.Set("Accept", "application/vnd.github.v3.raw") - } + resp, err := http.DefaultClient.Do(req) + if err != nil { + handleAPI(err, job, "Failed to get compose URL") + } + defer resp.Body.Close() + file, err := ioutil.ReadAll(resp.Body) + handleAPI(err, job, "Failed to read compose file") + + // Make directory for docker and logs and save file + os.Mkdir(folder.Docker+id, 0777) + os.Mkdir(logsLoc, 0777) + + if strings.Contains(string(file), "404") { + err = errors.New("404 in Compose File") + text := "Github Token invalid or wrong compose URL" + handleAPI(err, job, text) + job.Issue = text + } - resp, err := http.DefaultClient.Do(req) - if err != nil { - handleAPI(err, job, "Failed to get compose URL") - } - defer resp.Body.Close() - file, err := ioutil.ReadAll(resp.Body) - handleAPI(err, job, "Failed to read compose file") - - // Make directory for docker and logs and save file - os.Mkdir(folder.Docker+id, 0777) - os.Mkdir(logsLoc, 0777) - - if strings.Contains(string(file), "404") { - err = errors.New("404 in Compose File") - text := "Github Token invalid or wrong compose URL" - handleAPI(err, job, text) - job.Issue = text - } + // If Token was valid and compose file not empty + if err == nil { - // If Token was valid and compose file not empty + // Ensure ComposeLoc is Empty + _, err = os.Stat(composeLoc) if err == nil { + err = os.Remove(composeLoc) - // Ensure ComposeLoc is Empty - _, err = os.Stat(composeLoc) - if err == nil { - err = os.Remove(composeLoc) - - if err != nil { - handleAPI(err, job, "Failed to Remove") - } + if err != nil { + handleAPI(err, job, "Failed to Remove") } - - // Write to file - err = ioutil.WriteFile(composeLoc, file, 0777) - handleAPI(err, job, "Failed to write to file") } - if job.Registry != "" { - job.dockerLogin() - } + // Write to file + err = ioutil.WriteFile(composeLoc, file, 0777) + handleAPI(err, job, "Failed to write to file") + } - // Pull new images - job.buildCommand("-f", composeLoc, "pull") + if job.Registry != "" { + job.dockerLogin() + } - // Make sure config is valid - err = job.buildCommand("-f", composeLoc, "up", "--no-start") - if err == nil { - // Remove any previous containers - // Deals with any network active endpoints - job.buildCommand("-f", composeLoc, "down", "--remove-orphans") + // Pull new images + job.buildCommand("-f", composeLoc, "pull") - // Run Code - job.buildCommand("-f", composeLoc, "up", "-d") + // Make sure config is valid + err = job.buildCommand("-f", composeLoc, "up", "--no-start") + if err == nil { + // Remove any previous containers + // Deals with any network active endpoints + job.buildCommand("-f", composeLoc, "down", "--remove-orphans") - if job.Registry != "" { - job.dockerLogout() - } - } else { - return nil + // Run Code + job.buildCommand("-f", composeLoc, "up", "-d") + + if job.Registry != "" { + job.dockerLogout() } } - - // Get logging Running - cmd := exec.Command("docker-compose", "-f", composeLoc, "logs", "-f", "--no-color") - - outfile, err := os.Create(logsLoc + "/info.log") - handleAPI(err, job, "Failed to create log file") - cmd.Env = job.insertEnviroment() - cmd.Stdout = outfile - - return cmd } func (job *taskJob) insertEnviroment() []string { diff --git a/src/main.go b/src/main.go index e8ce3f1..a767c28 100644 --- a/src/main.go +++ b/src/main.go @@ -9,10 +9,12 @@ import ( ) var secretFlag string +var remoteFile string // Create Flags needed func init() { flag.StringVar(&secretFlag, "secret", "", "Put secret as a arg for automation tasks") + flag.StringVar(&remoteFile, "remotefile", "", "Location of Remote sharpcd.yml file") // Creates Helper Function flag.Usage = func() { diff --git a/src/server.go b/src/server.go index b37bf02..e93383f 100644 --- a/src/server.go +++ b/src/server.go @@ -62,7 +62,7 @@ func reconnect() { Reconnect: true} newJob.ID = item.Name() - allJobs.List = append(allJobs.List, &newJob) + allJobs[newJob.ID] = &newJob comm := &newJob go comm.Run() } diff --git a/src/structs.go b/src/structs.go index bd23d99..b60c4d7 100644 --- a/src/structs.go +++ b/src/structs.go @@ -4,7 +4,7 @@ import ( "github.com/gorilla/websocket" ) -var sharpCDVersion = "0.2.0" +var sharpCDVersion = "3" type statusCodes struct { NotPostMethod int @@ -36,6 +36,8 @@ var ( } ) +var allJobs map[string]*taskJob = make(map[string]*taskJob) + type jobStats struct { Running string Errored string @@ -62,10 +64,7 @@ type taskJob struct { Registry string `json:"registry"` Issue string `json:"issue"` Reconnect bool -} - -type allTaskJobs struct { - List []*taskJob + Kill bool } type postData struct { diff --git a/src/trak.go b/src/trak.go index 08c08ba..47eda70 100644 --- a/src/trak.go +++ b/src/trak.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "log" "net/http" + "os" "strconv" "strings" "time" @@ -54,7 +55,8 @@ func listJobs() { urlJobs := con.Trak[location] + "/api/jobs/" - jobs := getAPIOutput(urlJobs).Jobs + apiOutput, _ := getAPIOutput(urlJobs) + jobs := apiOutput.Jobs for _, job := range jobs { jobIDs = append(jobIDs, " - "+job.ID) @@ -95,7 +97,15 @@ func liveFeed() { } // Tests to ensure you can actually reach the server - getAPIOutput(urlJobs) + apiOutput, code := getAPIOutput(urlJobs) + + if code == statCode.Accepted { + fmt.Printf("Connection to API...") + } else { + fmt.Println(apiOutput.Message) + fmt.Printf("APi Connection Failed!\n") + os.Exit(1) + } // Load UI if err := ui.Init(); err != nil { @@ -111,14 +121,16 @@ func liveFeed() { // Gets all rows if there is one job if trakArg == trakOne { - job := getAPIOutput(urlJob).Job + apiOutput, _ := getAPIOutput(urlJob) + job := apiOutput.Job jobs = append(jobs, job) } // Gets all rows if there is there are multiple jobs if trakArg == trakAll { - jobs = getAPIOutput(urlJobs).Jobs + apiOutput, _ = getAPIOutput(urlJobs) + jobs = apiOutput.Jobs } // Adds in a row for each job @@ -262,13 +274,14 @@ func closing(width int, tablePadding int, heightLogs int) *widgets.Paragraph { // Creates the logging page func logging(width int, heightTable int, urlLog string) (*widgets.Paragraph, int) { - logs := getAPIOutput(urlLog).Message + logs, _ := getAPIOutput(urlLog) + msg := logs.Message deltaYLogs := heightTable heightLogs := deltaYLogs + 22 trakLogs := widgets.NewParagraph() - trakLogs.Text = "\n" + logs + trakLogs.Text = "\n" + msg trakLogs.SetRect(0, deltaYLogs, width, heightLogs) trakLogs.BorderStyle.Fg = ui.ColorCyan @@ -316,7 +329,7 @@ func generateColumnWidths(rows [][]string, columns []string) []int { return columnWidths } -func getAPIOutput(url string) response { +func getAPIOutput(url string) (response, int) { // Insert needed data secret := getSec() @@ -343,7 +356,7 @@ func getAPIOutput(url string) response { // Do Request resp, err := client.Do(req) - handle(err, "Failed to do POST request"+url) + handle(err, "Failed to connect to SharpCD Trak: "+url) defer resp.Body.Close() // Read Body and Status @@ -354,5 +367,5 @@ func getAPIOutput(url string) response { err = json.Unmarshal(body, &apiOutput) handle(err, "Failed to unmarshal body"+url) - return apiOutput + return apiOutput, resp.StatusCode } diff --git a/testing/restart.yml b/testing/restart.yml index e4cb5f6..920af80 100644 --- a/testing/restart.yml +++ b/testing/restart.yml @@ -6,5 +6,5 @@ services: hello_world2: image: ubuntu - command: ["/bin/sh", "-c", "echo Hello && sleep 3 && exit 2"] + command: ["/bin/sh", "-c", "echo Hello && sleep 1 && exit 2"] restart: always \ No newline at end of file