From 5d1bed68416b18efb2fa59ab6f70a15225caafa3 Mon Sep 17 00:00:00 2001 From: enyinna1234 Date: Tue, 30 Jul 2019 09:50:34 -0700 Subject: [PATCH] Adding Initial ICN API Service This adds create for adding images to the db, read for listing images, delete for removing images from the database, and update for changing image description. Signed-off-by: Enyinna Ochulor Change-Id: I8f524c2b5cfef6bc7749b2dc272574c937f24a03 --- cmd/bpa-restapi-agent/Makefile | 24 ++ cmd/bpa-restapi-agent/README.md | 6 + cmd/bpa-restapi-agent/api/api.go | 54 ++++ cmd/bpa-restapi-agent/api/imagehandler.go | 348 ++++++++++++++++++++++ cmd/bpa-restapi-agent/docs/swagger.yaml | 137 +++++++++ cmd/bpa-restapi-agent/go.mod | 16 + cmd/bpa-restapi-agent/go.sum | 25 ++ cmd/bpa-restapi-agent/internal/app/image.go | 194 ++++++++++++ cmd/bpa-restapi-agent/internal/config/config.go | 56 ++++ cmd/bpa-restapi-agent/internal/db/mongo.go | 379 ++++++++++++++++++++++++ cmd/bpa-restapi-agent/internal/db/store.go | 75 +++++ cmd/bpa-restapi-agent/internal/utils.go | 33 +++ cmd/bpa-restapi-agent/main.go | 54 ++++ cmd/bpa-restapi-agent/sample.json | 18 ++ 14 files changed, 1419 insertions(+) create mode 100644 cmd/bpa-restapi-agent/Makefile create mode 100644 cmd/bpa-restapi-agent/api/api.go create mode 100644 cmd/bpa-restapi-agent/api/imagehandler.go create mode 100644 cmd/bpa-restapi-agent/docs/swagger.yaml create mode 100644 cmd/bpa-restapi-agent/go.mod create mode 100644 cmd/bpa-restapi-agent/go.sum create mode 100644 cmd/bpa-restapi-agent/internal/app/image.go create mode 100644 cmd/bpa-restapi-agent/internal/config/config.go create mode 100644 cmd/bpa-restapi-agent/internal/db/mongo.go create mode 100644 cmd/bpa-restapi-agent/internal/db/store.go create mode 100644 cmd/bpa-restapi-agent/internal/utils.go create mode 100644 cmd/bpa-restapi-agent/main.go create mode 100644 cmd/bpa-restapi-agent/sample.json diff --git a/cmd/bpa-restapi-agent/Makefile b/cmd/bpa-restapi-agent/Makefile new file mode 100644 index 0000000..ba40a6f --- /dev/null +++ b/cmd/bpa-restapi-agent/Makefile @@ -0,0 +1,24 @@ + +# The name of the executable (default is current directory name) +TARGET := $(shell echo $${PWD\#\#*/}) +.DEFAULT_GOAL: $(TARGET) + +# These will be provided to the target +VERSION := 1.0.0 +BUILD := `git rev-parse HEAD` + +# Use linker flags to provide version/build settings to the target +LDFLAGS=-ldflags "-X=main.Version=$(VERSION) -X=main.Build=$(BUILD)" + +# go source files, ignore vendor directory +SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*") + +.PHONY: all build + +all: build + +$(TARGET): $(SRC) + @go build $(LDFLAGS) -o $(TARGET) + +build: $(TARGET) + @true diff --git a/cmd/bpa-restapi-agent/README.md b/cmd/bpa-restapi-agent/README.md index e69de29..a74434c 100644 --- a/cmd/bpa-restapi-agent/README.md +++ b/cmd/bpa-restapi-agent/README.md @@ -0,0 +1,6 @@ +### Running the server +To run the server, follow these simple steps: + +``` +go run main.go +``` diff --git a/cmd/bpa-restapi-agent/api/api.go b/cmd/bpa-restapi-agent/api/api.go new file mode 100644 index 0000000..d70cd80 --- /dev/null +++ b/cmd/bpa-restapi-agent/api/api.go @@ -0,0 +1,54 @@ +// api/api_images.go + + +package api + +import ( + image "bpa-restapi-agent/internal/app" + + "github.com/gorilla/mux" +) + +// NewRouter creates a router that registers the various urls that are supported +func NewRouter(binaryClient image.ImageManager, + containerClient image.ImageManager, + osClient image.ImageManager) *mux.Router { + + router := mux.NewRouter() + + //Setup the image uploaad api handler here + if binaryClient == nil { + binaryClient = image.NewBinaryImageClient() + } + binaryHandler := imageHandler{client: binaryClient} + imgRouter := router.PathPrefix("/v1").Subrouter() + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/binary_images", binaryHandler.createHandler).Methods("POST") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/binary_images/{imgname}", binaryHandler.getHandler).Methods("GET") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/binary_images/{imgname}", binaryHandler.deleteHandler).Methods("DELETE") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/binary_images/{imgname}", binaryHandler.updateHandler).Methods("PUT") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/binary_images/{imgname}", binaryHandler.patchHandler).Methods("PATCH") + + //Setup the _image upload api handler here + if containerClient == nil { + containerClient = image.NewContainerImageClient() + } + containerHandler := imageHandler{client: containerClient} + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/container_images", containerHandler.createHandler).Methods("POST") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/container_images/{imgname}", containerHandler.getHandler).Methods("GET") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/container_images/{imgname}", containerHandler.deleteHandler).Methods("DELETE") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/container_images/{imgname}", containerHandler.updateHandler).Methods("PUT") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/container_images/{imgname}", containerHandler.patchHandler).Methods("PATCH") + + //Setup the os_image upload api handler here + if osClient == nil { + osClient = image.NewOSImageClient() + } + osHandler := imageHandler{client: osClient} + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/os_images", osHandler.createHandler).Methods("POST") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/os_images/{imgname}", osHandler.getHandler).Methods("GET") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/os_images/{imgname}", osHandler.deleteHandler).Methods("DELETE") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/os_images/{imgname}", osHandler.updateHandler).Methods("PUT") + imgRouter.HandleFunc("/baremetalcluster/{owner}/{clustername}/os_images/{imgname}", osHandler.patchHandler).Methods("PATCH") + + return router +} diff --git a/cmd/bpa-restapi-agent/api/imagehandler.go b/cmd/bpa-restapi-agent/api/imagehandler.go new file mode 100644 index 0000000..0d7b787 --- /dev/null +++ b/cmd/bpa-restapi-agent/api/imagehandler.go @@ -0,0 +1,348 @@ +package api + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/user" + "log" + "path" + "strconv" + + image "bpa-restapi-agent/internal/app" + + "github.com/gorilla/mux" +) + +// imageHandler is used to store backend implementations objects +// Also simplifies mocking for unit testing purposes +type imageHandler struct { + // Interface that implements Image operations + // We will set this variable with a mock interface for testing + client image.ImageManager + dirPath string +} + +// CreateHandler handles creation of the image entry in the database + +func (h imageHandler) createHandler(w http.ResponseWriter, r *http.Request) { + var v image.Image + + // Implemenation using multipart form + // Review and enable/remove at a later date + // Set Max size to 16mb here + err := r.ParseMultipartForm(16777216) + if err != nil { + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + jsn := bytes.NewBuffer([]byte(r.FormValue("metadata"))) + err = json.NewDecoder(jsn).Decode(&v) + switch { + case err == io.EOF: + http.Error(w, "Empty body", http.StatusBadRequest) + return + case err != nil: + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + // Name is required. + if v.ImageName == "" { + http.Error(w, "Missing name in POST request", http.StatusBadRequest) + return + } + + // Owner is required. + if v.Owner == "" { + http.Error(w, "Missing Owner in POST request", http.StatusBadRequest) + return + } + + if v.ImageLength == 0 { + e := "Improper upload length" + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(e)) + return + } + + //Create file directory + dir, err := createFileDir(v.Type) + if err != nil { + log.Fatal("Error creating file server directory", err) + } + + //Read the file section and ignore the header + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, "Unable to process file", http.StatusUnprocessableEntity) + return + } + + defer file.Close() + + //Convert the file content to base64 for storage + content, err := ioutil.ReadAll(file) + if err != nil { + http.Error(w, "Unable to read file", http.StatusUnprocessableEntity) + return + } + + v.Config = base64.StdEncoding.EncodeToString(content) + + ret, err := h.client.Create(v) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + h.dirPath = dir + filePath := path.Join(h.dirPath, v.ImageName) + file1, err := os.Create(filePath) + if err != nil { + e := "Error creating file in filesystem" + log.Printf("%s %s\n", e, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + defer file1.Close() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + err = json.NewEncoder(w).Encode(ret) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// Create file + +func createFileDir(dirName string) (string, error) { + u, err := user.Current() + if err != nil { + log.Println("Error while fetching user home directory", err) + return "", err + } + home := u.HomeDir + dirPath := path.Join(home, "images", dirName) + err = os.MkdirAll(dirPath, 0744) + if err != nil { + log.Println("Error while creating file server directory", err) + return "", err + } + return dirPath, nil +} + +// getHandler handles GET operations on a particular name +// Returns an Image +func (h imageHandler) getHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + // ownerName := vars["owner"] + // clusterName := vars["clustername"] + imageName := vars["imgname"] + + ret, err := h.client.Get(imageName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + err = json.NewEncoder(w).Encode(ret) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// deleteHandler handles DELETE operations on a particular record +func (h imageHandler) deleteHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + // ownerName := vars["owner"] + // clusterName := vars["clustername"] + imageName := vars["imgname"] + + err := h.client.Delete(imageName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// UpdateHandler handles Update operations on a particular image +func (h imageHandler) updateHandler(w http.ResponseWriter, r *http.Request) { + var v image.Image + vars := mux.Vars(r) + imageName := vars["imgname"] + + err := r.ParseMultipartForm(16777216) + if err != nil { + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + jsn := bytes.NewBuffer([]byte(r.FormValue("metadata"))) + err = json.NewDecoder(jsn).Decode(&v) + switch { + case err == io.EOF: + http.Error(w, "Empty body", http.StatusBadRequest) + return + case err != nil: + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + + // Name is required. + if v.ImageName == "" { + http.Error(w, "Missing name in PUT request", http.StatusBadRequest) + return + } + + // Owner is required. + if v.Owner == "" { + http.Error(w, "Missing Owner in PUT request", http.StatusBadRequest) + return + } + + //Read the file section and ignore the header + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, "Unable to process file", http.StatusUnprocessableEntity) + return + } + + defer file.Close() + + //Convert the file content to base64 for storage + content, err := ioutil.ReadAll(file) + if err != nil { + http.Error(w, "Unable to read file", http.StatusUnprocessableEntity) + return + } + + v.Config = base64.StdEncoding.EncodeToString(content) + + ret, err := h.client.Update(imageName, v) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + err = json.NewEncoder(w).Encode(ret) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// File upload is handled by the patchHandler + +func (h imageHandler) patchHandler(w http.ResponseWriter, r *http.Request) { + log.Println("going to patch file") + vars := mux.Vars(r) + imageName := vars["imgname"] + file, err := h.client.Get(imageName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if *file.UploadComplete == true { + e := "Upload already completed" + w.WriteHeader(http.StatusUnprocessableEntity) + w.Write([]byte(e)) + return + } + off, err := strconv.Atoi(r.Header.Get("Upload-Offset")) + if err != nil { + log.Println("Improper upload offset", err) + w.WriteHeader(http.StatusBadRequest) + return + } + log.Printf("Upload offset %d\n", off) + if *file.ImageOffset != off { + e := fmt.Sprintf("Expected Offset %d got offset %d", *file.ImageOffset, off) + w.WriteHeader(http.StatusConflict) + w.Write([]byte(e)) + return + } + + log.Println("Content length is", r.Header.Get("Content-Length")) + clh := r.Header.Get("Content-Length") + cl, err := strconv.Atoi(clh) + if err != nil { + log.Println("unknown content length") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if cl != (file.ImageLength - *file.ImageOffset) { + e := fmt.Sprintf("Content length doesn't match upload length. Expected content length %d got %d", file.ImageLength-*file.ImageOffset, cl) + log.Println(e) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(e)) + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("Received file partially %s\n", err) + log.Println("Size of received file ", len(body)) + } + + u, err := user.Current() + if err != nil { + log.Println("Error while fetching user home directory", err) + return + } + home := u.HomeDir + dir := path.Join(home, "images", file.Type) + h.dirPath = dir + fp := fmt.Sprintf("%s/%s", h.dirPath, imageName) + f, err := os.OpenFile(fp, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + log.Printf("unable to open file %s\n", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + defer f.Close() + + n, err := f.WriteAt(body, int64(off)) + if err != nil { + log.Printf("unable to write %s", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + log.Println("number of bytes written ", n) + no := *file.ImageOffset + n + file.ImageOffset = &no + + uo := strconv.Itoa(*file.ImageOffset) + w.Header().Set("Upload-Offset", uo) + if *file.ImageOffset == file.ImageLength { + log.Println("upload completed successfully") + *file.UploadComplete = true + } + + // err = h.updateFile(file) + // if err != nil { + // log.Println("Error while updating file", err) + // w.WriteHeader(http.StatusInternalServerError) + // return + // } + w.WriteHeader(http.StatusNoContent) + + return + +} diff --git a/cmd/bpa-restapi-agent/docs/swagger.yaml b/cmd/bpa-restapi-agent/docs/swagger.yaml new file mode 100644 index 0000000..a375700 --- /dev/null +++ b/cmd/bpa-restapi-agent/docs/swagger.yaml @@ -0,0 +1,137 @@ +--- +swagger: "2.0" +info: + description: "Addresses deployment of workloads in the edge" + version: "1.0.0" + title: "ICN application" +schemes: +- "http" +consumes: +- "application/json" +produces: +- "application/json" +paths: + /: + get: + tags: + - "container_images" + operationId: "find_images" + parameters: + - name: "since" + in: "query" + required: false + type: "integer" + format: "int64" + x-exportParamName: "Since" + x-optionalDataType: "Int64" + - name: "limit" + in: "query" + required: false + type: "integer" + default: 20 + format: "int32" + x-exportParamName: "Limit" + x-optionalDataType: "Int32" + responses: + 200: + description: "list the ICN operations" + schema: + type: "array" + items: + $ref: "#/definitions/Request" + default: + description: "generic error response" + schema: + $ref: "#/definitions/error" + post: + tags: + - "container_images" + operationId: "addContainer" + parameters: + - in: "body" + name: "body" + required: false + schema: + $ref: "#/definitions/Request" + x-exportParamName: "Body" + responses: + 201: + description: "Created" + schema: + $ref: "#/definitions/Request" + default: + description: "error" + schema: + $ref: "#/definitions/error" + /{id}: + put: + tags: + - "container_images" + operationId: "updateImage" + parameters: + - name: "id" + in: "path" + required: true + type: "integer" + format: "int64" + x-exportParamName: "Id" + - in: "body" + name: "body" + required: false + schema: + $ref: "#/definitions/Request" + x-exportParamName: "Body" + responses: + 200: + description: "OK" + schema: + $ref: "#/definitions/Request" + default: + description: "error" + schema: + $ref: "#/definitions/error" + delete: + tags: + - "container_images" + operationId: "destroyImage" + parameters: + - name: "id" + in: "path" + required: true + type: "integer" + format: "int64" + x-exportParamName: "Id" + responses: + 204: + description: "Deleted" + default: + description: "error" + schema: + $ref: "#/definitions/error" +definitions: + Request: + type: "object" + properties: + image_id: + type: "string" + repo: + type: "string" + tag: + type: "string" + installed: + type: "boolean" + example: + installed: true + repo: "repo" + tag: "tag" + image_id: "image_id" + error: + type: "object" + required: + - "message" + properties: + code: + type: "integer" + format: "int64" + message: + type: "string" diff --git a/cmd/bpa-restapi-agent/go.mod b/cmd/bpa-restapi-agent/go.mod new file mode 100644 index 0000000..5f99a12 --- /dev/null +++ b/cmd/bpa-restapi-agent/go.mod @@ -0,0 +1,16 @@ +module bpa-restapi-agent + +go 1.12 + +require ( + github.com/go-stack/stack v1.8.0 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/gorilla/handlers v1.4.2 + github.com/gorilla/mux v1.7.3 + github.com/pkg/errors v0.8.1 + github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect + github.com/xdg/stringprep v1.0.0 // indirect + go.mongodb.org/mongo-driver v1.0.4 + golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect +) diff --git a/cmd/bpa-restapi-agent/go.sum b/cmd/bpa-restapi-agent/go.sum new file mode 100644 index 0000000..50f7957 --- /dev/null +++ b/cmd/bpa-restapi-agent/go.sum @@ -0,0 +1,25 @@ +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +go.mongodb.org/mongo-driver v1.0.4 h1:bHxbjH6iwh1uInchXadI6hQR107KEbgYsMzoblDONmQ= +go.mongodb.org/mongo-driver v1.0.4/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/cmd/bpa-restapi-agent/internal/app/image.go b/cmd/bpa-restapi-agent/internal/app/image.go new file mode 100644 index 0000000..a1beed8 --- /dev/null +++ b/cmd/bpa-restapi-agent/internal/app/image.go @@ -0,0 +1,194 @@ +package app + +import ( + //"encoding/base64" + "encoding/json" + //"io/ioutil" + + "bpa-restapi-agent/internal/db" + + pkgerrors "github.com/pkg/errors" +) + +// Image contains the parameters needed for Image information +type Image struct { + Owner string `json:"owner"` + ClusterName string `json:"cluster_name"` + Type string `json:"type"` + ImageName string `json:"image_name"` + Config string `json:"config"` + ImageOffset *int `json:"image_offset"` + ImageLength int `json:"image_length"` + UploadComplete *bool `json:"upload_complete"` + Description ImageRecordList `json:"description"` +} + +type ImageRecordList struct { + ImageRecords []map[string]string `json:"image_records"` +} + +// ImageKey is the key structure that is used in the database +type ImageKey struct { + // Owner string `json:"owner"` + // ClusterName string `json:"cluster_name"` + ImageName string `json:"image_name"` +} + +// We will use json marshalling to convert to string to +// preserve the underlying structure. +func (dk ImageKey) String() string { + out, err := json.Marshal(dk) + if err != nil { + return "" + } + + return string(out) +} + +// ImageManager is an interface that exposes the Image functionality +type ImageManager interface { + Create(c Image) (Image, error) + Get(imageName string) (Image, error) + Delete(imageName string) error + Update(imageName string, c Image) (Image, error) + GetImageRecordByName(imgname, imageName string) (map[string]string, error) +} + +// ImageClient implements the ImageManager +// It will also be used to maintain some localized state +type ImageClient struct { + storeName string + tagMeta string +} + +// To Do - Fix repetition in +// NewImageClient returns an instance of the ImageClient +// which implements the ImageManager +func NewBinaryImageClient() *ImageClient { + return &ImageClient{ + storeName: "binary_image", + tagMeta: "metadata", + } +} + +func NewContainerImageClient() *ImageClient { + return &ImageClient{ + storeName: "container_image", + tagMeta: "metadata", + } +} + +func NewOSImageClient() *ImageClient { + return &ImageClient{ + storeName: "os_image", + tagMeta: "metadata", + } +} + +// Create an entry for the Image resource in the database` +func (v *ImageClient) Create(c Image) (Image, error) { + + //Construct composite key consisting of name + key := ImageKey{ + // Owner: c.Owner, + // ClusterName: c.ClusterName, + ImageName: c.ImageName, + } + + //Check if this Image already exists + _, err := v.Get(c.ImageName) + if err == nil { + return Image{}, pkgerrors.New("Image already exists") + } + + err = db.DBconn.Create(v.storeName, key, v.tagMeta, c) + if err != nil { + return Image{}, pkgerrors.Wrap(err, "Creating DB Entry") + } + + return c, nil +} + +// Get returns Image for corresponding to name +func (v *ImageClient) Get(imageName string) (Image, error) { + + //Construct the composite key to select the entry + key := ImageKey{ + // Owner: ownerName, + // ClusterName: clusterName, + ImageName: imageName, + } + + value, err := db.DBconn.Read(v.storeName, key, v.tagMeta) + if err != nil { + return Image{}, pkgerrors.Wrap(err, "Get Image") + } + + //value is a byte array + if value != nil { + c := Image{} + err = db.DBconn.Unmarshal(value, &c) + if err != nil { + return Image{}, pkgerrors.Wrap(err, "Unmarshaling Value") + } + return c, nil + } + + return Image{}, pkgerrors.New("Error getting Connection") +} + +func (v *ImageClient) GetImageRecordByName(imgName string, + imageRecordName string) (map[string]string, error) { + + img, err := v.Get(imgName) + if err != nil { + return nil, pkgerrors.Wrap(err, "Error getting image") + } + + for _, value := range img.Description.ImageRecords { + if imageRecordName == value["image_record_name"] { + return value, nil + } + } + + return nil, pkgerrors.New("Image record " + imageRecordName + " not found") +} + +// Delete the Image from database +func (v *ImageClient) Delete(imageName string) error { + + //Construct the composite key to select the entry + key := ImageKey{ + // Owner: ownerName, + // ClusterName: clusterName, + ImageName: imageName, + } + err := db.DBconn.Delete(v.storeName, key, v.tagMeta) + if err != nil { + return pkgerrors.Wrap(err, "Delete Image") + } + return nil +} + +// Update an entry for the image in the database +func (v *ImageClient) Update(imageName string, c Image) (Image, error) { + + key := ImageKey{ + // Owner: c.Owner, + // ClusterName: c.ClusterName, + ImageName: imageName, + } + + //Check if this Image exists + _, err := v.Get(imageName) + if err != nil { + return Image{}, pkgerrors.New("Update Error - Image doesn't exist") + } + + err = db.DBconn.Update(v.storeName, key, v.tagMeta, c) + if err != nil { + return Image{}, pkgerrors.Wrap(err, "Updating DB Entry") + } + + return c, nil +} diff --git a/cmd/bpa-restapi-agent/internal/config/config.go b/cmd/bpa-restapi-agent/internal/config/config.go new file mode 100644 index 0000000..e5d4f48 --- /dev/null +++ b/cmd/bpa-restapi-agent/internal/config/config.go @@ -0,0 +1,56 @@ +package config + +import ( + "encoding/json" + "log" + "os" +) + +type Configuration struct { + Password string `json: "password"` + DatabaseAddress string `json: "database-address"` + DatabaseType string `json: "database-type"` + ServicePort string `json: "service-port"` +} + +var gConfig *Configuration + +func readConfigFile(file string) (*Configuration, error) { + f, err := os.Open(file) + if err != nil { + return defaultConfiguration(), err + } + defer f.Close() + + conf := defaultConfiguration() + + decoder := json.NewDecoder(f) + err = decoder.Decode(conf) + if err != nil { + return conf, err + } + + return conf, nil +} + +func defaultConfiguration() *Configuration { + return &Configuration { + Password: "", + DatabaseAddress: "127.0.0.1", + DatabaseType: "mongo", + ServicePort: "9015", + } +} + +func GetConfiguration() *Configuration { + if gConfig == nil { + conf, err := readConfigFile("ICNconfig.json") + if err != nil { + log.Println("Error loading config file. Using defaults") + } + + gConfig = conf + } + + return gConfig +} diff --git a/cmd/bpa-restapi-agent/internal/db/mongo.go b/cmd/bpa-restapi-agent/internal/db/mongo.go new file mode 100644 index 0000000..454f26c --- /dev/null +++ b/cmd/bpa-restapi-agent/internal/db/mongo.go @@ -0,0 +1,379 @@ +package db + +import ( + "golang.org/x/net/context" + "log" + + "bpa-restapi-agent/internal/config" + + pkgerrors "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// MongoCollection defines the a subset of MongoDB operations +// Note: This interface is defined mainly for mock testing +type MongoCollection interface { + InsertOne(ctx context.Context, document interface{}, + opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) + FindOne(ctx context.Context, filter interface{}, + opts ...*options.FindOneOptions) *mongo.SingleResult + FindOneAndUpdate(ctx context.Context, filter interface{}, + update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult + DeleteOne(ctx context.Context, filter interface{}, + opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) + Find(ctx context.Context, filter interface{}, + opts ...*options.FindOptions) (*mongo.Cursor, error) +} + +// MongoStore is an implementation of the db.Store interface +type MongoStore struct { + db *mongo.Database +} + +// This exists only for allowing us to mock the collection object +// for testing purposes +var getCollection = func(coll string, m *MongoStore) MongoCollection { + return m.db.Collection(coll) +} + +// This exists only for allowing us to mock the DecodeBytes function +// Mainly because we cannot construct a SingleResult struct from our +// tests. All fields in that struct are private. +var decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { + return sr.DecodeBytes() +} + +// These exists only for allowing us to mock the cursor.Next function +// Mainly because we cannot construct a mongo.Cursor struct from our +// tests. All fields in that struct are private and there is no public +// constructor method. +var cursorNext = func(ctx context.Context, cursor *mongo.Cursor) bool { + return cursor.Next(ctx) +} +var cursorClose = func(ctx context.Context, cursor *mongo.Cursor) error { + return cursor.Close(ctx) +} + +// NewMongoStore initializes a Mongo Database with the name provided +// If a database with that name exists, it will be returned +func NewMongoStore(name string, store *mongo.Database) (Store, error) { + if store == nil { + ip := "mongodb://" + config.GetConfiguration().DatabaseAddress + ":27017" + clientOptions := options.Client() + clientOptions.ApplyURI(ip) + mongoClient, err := mongo.NewClient(clientOptions) + if err != nil { + return nil, err + } + + err = mongoClient.Connect(context.Background()) + if err != nil { + return nil, err + } + store = mongoClient.Database(name) + } + + return &MongoStore{ + db: store, + }, nil +} + +// HealthCheck verifies if the database is up and running +func (m *MongoStore) HealthCheck() error { + + _, err := decodeBytes(m.db.RunCommand(context.Background(), bson.D{{"serverStatus", 1}})) + if err != nil { + return pkgerrors.Wrap(err, "Error getting server status") + } + + return nil +} + +// validateParams checks to see if any parameters are empty +func (m *MongoStore) validateParams(args ...interface{}) bool { + for _, v := range args { + val, ok := v.(string) + if ok { + if val == "" { + return false + } + } else { + if v == nil { + return false + } + } + } + + return true +} + +// Create is used to create a DB entry +func (m *MongoStore) Create(coll string, key Key, tag string, data interface{}) error { + if data == nil || !m.validateParams(coll, key, tag) { + return pkgerrors.New("No Data to store") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Insert the data and then add the objectID to the masterTable + res, err := c.InsertOne(ctx, bson.D{ + {tag, data}, + }) + if err != nil { + return pkgerrors.Errorf("Error inserting into database: %s", err.Error()) + } + + //Add objectID of created data to masterKey document + //Create masterkey document if it does not exist + filter := bson.D{{"key", key}} + + _, err = decodeBytes( + c.FindOneAndUpdate( + ctx, + filter, + bson.D{ + {"$set", bson.D{ + {tag, res.InsertedID}, + }}, + }, + options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After))) + + if err != nil { + return pkgerrors.Errorf("Error updating master table: %s", err.Error()) + } + + return nil +} + +// Update is used to update a DB entry +func (m *MongoStore) Update(coll string, key Key, tag string, data interface{}) error { + if data == nil || !m.validateParams(coll, key, tag) { + return pkgerrors.New("No Data to update") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) + if err != nil { + return pkgerrors.Errorf("Error finding master table: %s", err.Error()) + } + + //Read the tag objectID from document + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Update the document with new data + filter = bson.D{{"_id", tagoid}} + + _, err = decodeBytes( + c.FindOneAndUpdate( + ctx, + filter, + bson.D{ + {"$set", bson.D{ + {tag, data}, + }}, + }, + options.FindOneAndUpdate().SetReturnDocument(options.After))) + + if err != nil { + return pkgerrors.Errorf("Error updating record: %s", err.Error()) + } + + return nil +} + +// Unmarshal implements an unmarshaler for bson data that +// is produced from the mongo database +func (m *MongoStore) Unmarshal(inp []byte, out interface{}) error { + err := bson.Unmarshal(inp, out) + if err != nil { + return pkgerrors.Wrap(err, "Unmarshaling bson") + } + return nil +} + +// Read method returns the data stored for this key and for this particular tag +func (m *MongoStore) Read(coll string, key Key, tag string) ([]byte, error) { + if !m.validateParams(coll, key, tag) { + return nil, pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error finding master table: %s", err.Error()) + } + + //Read the tag objectID from document + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return nil, pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + filter = bson.D{{"_id", tagoid}} + tagdata, err := decodeBytes(c.FindOne(ctx, filter)) + if err != nil { + return nil, pkgerrors.Errorf("Error reading found object: %s", err.Error()) + } + + //Return the data as a byte array + //Convert string data to byte array using the built-in functions + switch tagdata.Lookup(tag).Type { + case bson.TypeString: + return []byte(tagdata.Lookup(tag).StringValue()), nil + default: + return tagdata.Lookup(tag).Value, nil + } +} + +// Helper function that deletes an object by its ID +func (m *MongoStore) deleteObjectByID(coll string, objID primitive.ObjectID) error { + + c := getCollection(coll, m) + ctx := context.Background() + + _, err := c.DeleteOne(ctx, bson.D{{"_id", objID}}) + if err != nil { + return pkgerrors.Errorf("Error Deleting from database: %s", err.Error()) + } + + log.Printf("Deleted Obj with ID %s", objID.String()) + return nil +} + +// Delete method removes a document from the Database that matches key +// TODO: delete all referenced docs if tag is empty string +func (m *MongoStore) Delete(coll string, key Key, tag string) error { + if !m.validateParams(coll, key, tag) { + return pkgerrors.New("Mandatory fields are missing") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get the masterkey document based on given key + filter := bson.D{{"key", key}} + //Remove the tag ID entry from masterkey table + update := bson.D{ + { + "$unset", bson.D{ + {tag, ""}, + }, + }, + } + keydata, err := decodeBytes(c.FindOneAndUpdate(ctx, filter, update, + options.FindOneAndUpdate().SetReturnDocument(options.Before))) + if err != nil { + //No document was found. Return nil. + if err == mongo.ErrNoDocuments { + return nil + } + //Return any other error that was found. + return pkgerrors.Errorf("Error decoding master table after update: %s", + err.Error()) + } + + //Read the tag objectID from document + elems, err := keydata.Elements() + if err != nil { + return pkgerrors.Errorf("Error reading elements from database: %s", err.Error()) + } + + tagoid, ok := keydata.Lookup(tag).ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for tag %s", tag) + } + + //Use tag objectID to read the data from store + err = m.deleteObjectByID(coll, tagoid) + if err != nil { + return pkgerrors.Errorf("Error deleting from database: %s", err.Error()) + } + + //Delete master table if no more tags left + //_id, key and tag should be elements in before doc + //if master table needs to be removed too + if len(elems) == 3 { + keyid, ok := keydata.Lookup("_id").ObjectIDOK() + if !ok { + return pkgerrors.Errorf("Error finding objectID for key %s", key) + } + err = m.deleteObjectByID(coll, keyid) + if err != nil { + return pkgerrors.Errorf("Error deleting master table from database: %s", err.Error()) + } + } + + return nil +} + +// ReadAll is used to get all documents in db of a particular tag +func (m *MongoStore) ReadAll(coll, tag string) (map[string][]byte, error) { + if !m.validateParams(coll, tag) { + return nil, pkgerrors.New("Missing collection or tag name") + } + + c := getCollection(coll, m) + ctx := context.Background() + + //Get all master tables in this collection + filter := bson.D{ + {"key", bson.D{ + {"$exists", true}, + }}, + } + cursor, err := c.Find(ctx, filter) + if err != nil { + return nil, pkgerrors.Errorf("Error reading from database: %s", err.Error()) + } + defer cursorClose(ctx, cursor) + + //Iterate over all the master tables + result := make(map[string][]byte) + for cursorNext(ctx, cursor) { + d := cursor.Current + + //Read key of each master table + key, ok := d.Lookup("key").DocumentOK() + if !ok { + //Throw error if key is not found + pkgerrors.New("Unable to read key from mastertable") + } + + //Get objectID of tag document + tid, ok := d.Lookup(tag).ObjectIDOK() + if !ok { + log.Printf("Did not find tag: %s", tag) + continue + } + + //Find tag document and unmarshal it into []byte + tagData, err := decodeBytes(c.FindOne(ctx, bson.D{{"_id", tid}})) + if err != nil { + log.Printf("Unable to decode tag data %s", err.Error()) + continue + } + result[key.String()] = tagData.Lookup(tag).Value + } + + if len(result) == 0 { + return result, pkgerrors.Errorf("Did not find any objects with tag: %s", tag) + } + + return result, nil +} diff --git a/cmd/bpa-restapi-agent/internal/db/store.go b/cmd/bpa-restapi-agent/internal/db/store.go new file mode 100644 index 0000000..0b981e7 --- /dev/null +++ b/cmd/bpa-restapi-agent/internal/db/store.go @@ -0,0 +1,75 @@ +package db + +import ( + "encoding/json" + "reflect" + + pkgerrors "github.com/pkg/errors" +) + +// DBconn interface used to talk to a concrete Database connection +var DBconn Store + +// Key is an interface that will be implemented by anypackage +// that wants to use the Store interface. This allows various +// db backends and key types. +type Key interface { + String() string +} + +// Store is an interface for accessing a database +type Store interface { + // Returns nil if db health is good + HealthCheck() error + + // Unmarshal implements any unmarshaling needed for the database + Unmarshal(inp []byte, out interface{}) error + + // Creates a new master table with key and links data with tag and + // creates a pointer to the newly added data in the master table + Create(table string, key Key, tag string, data interface{}) error + + // Reads data for a particular key with specific tag. + Read(table string, key Key, tag string) ([]byte, error) + + // Update data for particular key with specific tag + Update(table string, key Key, tag string, data interface{}) error + + // Deletes a specific tag data for key. + // TODO: If tag is empty, it will delete all tags under key. + Delete(table string, key Key, tag string) error + + // Reads all master tables and data from the specified tag in table + ReadAll(table string, tag string) (map[string][]byte, error) +} + +// CreateDBClient creates the DB client +func CreateDBClient(dbType string) error { + var err error + switch dbType { + case "mongo": + // create a mongodb database with ICN as the name + DBconn, err = NewMongoStore("icn", nil) + default: + return pkgerrors.New(dbType + "DB not supported") + } + return err +} + +// Serialize converts given data into a JSON string +func Serialize(v interface{}) (string, error) { + out, err := json.Marshal(v) + if err != nil { + return "", pkgerrors.Wrap(err, "Error serializing "+reflect.TypeOf(v).String()) + } + return string(out), nil +} + +// DeSerialize converts string to a json object specified by type +func DeSerialize(str string, v interface{}) error { + err := json.Unmarshal([]byte(str), &v) + if err != nil { + return pkgerrors.Wrap(err, "Error deSerializing "+str) + } + return nil +} diff --git a/cmd/bpa-restapi-agent/internal/utils.go b/cmd/bpa-restapi-agent/internal/utils.go new file mode 100644 index 0000000..b590789 --- /dev/null +++ b/cmd/bpa-restapi-agent/internal/utils.go @@ -0,0 +1,33 @@ +package utils + +import( + //"log" + "bpa-restapi-agent/internal/db" + "bpa-restapi-agent/internal/config" + pkgerrors "github.com/pkg/errors" +) + +func CheckDatabaseConnection() error { +// To Do - Implement db and config + + err := db.CreateDBClient(config.GetConfiguration().DatabaseType) + if err != nil { + return pkgerrors.Cause(err) + } + + err = db.DBconn.HealthCheck() + if err != nil { + return pkgerrors.Cause(err) + } + + return nil +} + +func CheckInitialSettings() error { + err := CheckDatabaseConnection() + if err != nil { + return pkgerrors.Cause(err) + } + + return nil +} diff --git a/cmd/bpa-restapi-agent/main.go b/cmd/bpa-restapi-agent/main.go new file mode 100644 index 0000000..6a8960b --- /dev/null +++ b/cmd/bpa-restapi-agent/main.go @@ -0,0 +1,54 @@ +// main.go +package main + +import ( + "context" + "log" + "math/rand" + "net/http" + "os" + "os/signal" + "time" + + //To Do - Implement internal for checking config + "github.com/gorilla/handlers" + + "bpa-restapi-agent/api" + utils "bpa-restapi-agent/internal" + "bpa-restapi-agent/internal/config" +) + +func main() { + // To Do - Implement initial settings + // check initial config + err := utils.CheckInitialSettings() + if err != nil{ + log.Fatal(err) + } + + rand.Seed(time.Now().UnixNano()) + + httpRouter := api.NewRouter(nil, nil, nil) + // Return http.handler and log requests to Stdout + loggedRouter := handlers.LoggingHandler(os.Stdout, httpRouter) + log.Println("Starting Integrated Cloud Native API") + + // Create custom http server + httpServer := &http.Server{ + Handler: loggedRouter, + // To Do - Implement config + Addr: ":" + config.GetConfiguration().ServicePort, + } + connectionsClose := make(chan struct{}) + go func() { + c := make(chan os.Signal, 1) // create c channel to receive notifications + signal.Notify(c, os.Interrupt) // register c channel to run concurrently + <-c + httpServer.Shutdown(context.Background()) + close(connectionsClose) + }() + + // Start server + log.Fatal(httpServer.ListenAndServe()) + +} diff --git a/cmd/bpa-restapi-agent/sample.json b/cmd/bpa-restapi-agent/sample.json new file mode 100644 index 0000000..97c2125 --- /dev/null +++ b/cmd/bpa-restapi-agent/sample.json @@ -0,0 +1,18 @@ +{ + "owner": "alpha", + "cluster_name": "beta", + "type": "container", + "image_name": "asdf246", + "image_length": 21579557, + "image_offset": 0, + "upload_complete": false, + "description": { + "image_records": [ + { + "image_record_name": "iuysdi1234", + "repo": "java", + "tag": "8" + } + ] + } +} -- 2.16.6