From: Todd Malsbary Date: Mon, 24 Jan 2022 22:42:52 +0000 (-0800) Subject: Remove BPA REST API X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=01f9915e00d5e886df00e1d9770c53eeac445059;p=icn.git Remove BPA REST API Signed-off-by: Todd Malsbary Change-Id: I5863fc4b8d2517fb1ecbff7d3fa6c16f90fafcec --- diff --git a/cmd/bpa-restapi-agent/Makefile b/cmd/bpa-restapi-agent/Makefile deleted file mode 100644 index 6437dc3..0000000 --- a/cmd/bpa-restapi-agent/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -.PHONY: build - -build: - go build -o build/_output/bin/bpa-restapi-agent main.go - -docker: - docker build -t akraino.org/icn/bpa-restapi-agent:latest . -f build/Dockerfile - -deploy: docker - ./bpa_api_install.sh - -unit_test: go_install - go test ./internal/app - -go_install: - ./install_go.sh - -e2e_test: deploy - ./e2e_test.sh - -clean: - ./bpa_api_uninstall.sh diff --git a/cmd/bpa-restapi-agent/README.md b/cmd/bpa-restapi-agent/README.md deleted file mode 100644 index a3300c0..0000000 --- a/cmd/bpa-restapi-agent/README.md +++ /dev/null @@ -1,211 +0,0 @@ -### Running the server -To run the server, follow these simple steps: - -Integrated Cloud Native (ICN) RESTful API - -This is a Golang application providing a RESTful API to interact with and upload image objects. - -The API application source files are in the icn/cmd/bpa-restapi-agent directory. - -While the database back-end is extensible, this initial release requires mongodb. - -Install - -Install and start mongodb. For instructions: https://docs.mongodb.com/manual/installation/ - -git clone "https://gerrit.akraino.org/r/icn" -cd icn/cmd/bpa-restapi-agent - -Run the application -go run main.go - -Output without a config file: - -2019/08/22 14:08:41 Error loading config file. Using defaults -2019/08/22 14:08:41 Starting Integrated Cloud Native API - -RESTful API usage examples - -Sample Post Request - -curl -i -F "metadata=//jsonfile -X POST http://NODE_IP:9015//baremetalcluster/{owner}/{clustername}/ - -#image type can be binary_image, container_image, or os_image - -Example requests and responses: - -Create image - POST - -#Using a json file called sample.json -#image_length in sample.json can be determined with the command -ls -al - -Request - -curl -i -F "metadata=/{imgname} - - -example: -#continuing with our container image from above - -Request - -curl -i -X GET http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 - -Response - -HTTP/1.1 200 OK -Content-Type: application/json -Date: Thu, 22 Aug 2019 22:57:10 GMT -Content-Length: 239 - -{"owner":"alpha","cluster_name":"beta","type":"container","image_name":"asdf246","image_offset":0,"image_length":29718177,"upload_complete":false,"description":{"image_records":[{"image_record_name":"iuysdi1234","repo":"icn","tag":"1"}]}} - -Upload container image - PATCH -Request - -curl --request PATCH --data-binary "@/home/enyi/workspace/icn/cmd/bpa-restapi-agent/sample_image" http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 --header "Upload-Offset: 0" --header "Expect:" -i - - -Response - -HTTP/1.1 204 No Content -Upload-Offset: 29718177 -Date: Thu, 22 Aug 2019 23:19:44 GMT - -Check uploaded image - GET - -Request - -curl -i -X GET http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 - -Response - -HTTP/1.1 200 OK -Content-Type: application/json -Date: Fri, 23 Aug 2019 17:12:07 GMT -Content-Length: 245 - -{"owner":"alpha","cluster_name":"beta","type":"container","image_name":"asdf246","image_offset":29718177,"image_length":29718177,"upload_complete":true,"description":{"image_records":[{"image_record_name":"iuysdi1234","repo":"icn","tag":"1"}]}} - -#after the upload, the image_offset is now the same as image_length and upload_complete changed to true -#if upload was incomplete - -Resumable upload instructions - -Resumable upload -PATCH - -#this is the current resumable upload mechanism - -Request - -curl --request PATCH --data-binary "@/home/enyi/workspace/icn/cmd/bpa-restapi-agent/sample_image" http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 --header "Upload-Offset: 0" --header "Expect:" -i --limit-rate 200K - -#the above request limits transfer for testing purposes -#'ctl c' out after a few seconds, to stop file transfer -#check image_offset with a GET - -Check upload - GET - -Request - -curl -i -X GET http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 - -Response - -HTTP/1.1 200 OK -Content-Type: application/json -Date: Sat, 24 Aug 2019 00:30:00 GMT -Content-Length: 245 - -{"owner":"alpha","cluster_name":"beta","type":"container","image_name":"asdf246","image_offset":4079616,"image_length":29718177,"upload_complete":false,"description":{"image_records":[{"image_record_name":"iuysdi1234","repo":"icn","tag":"2"}]}} - -#from our response you can see that image_offset is still less than image_length and #upload_complete is still false -#next we use the dd command (no limiting this time) - -Request - -dd if=/home/enyi/workspace/icn/cmd/bpa-restapi-agent/sample_image skip=4079616 bs=1 | curl --request PATCH --data-binary @- http://localhost:9015/v1/baremetalcluster/alpha/beta/container_images/asdf246 --header "Upload-Offset: 4079616" --header "Expect:" -i - -#the request skips already uploaded 4079616 bytes of data - -Response - -25638561+0 records in -25638561+0 records out -25638561 bytes (26 MB, 24 MiB) copied, 207.954 s, 123 kB/s -HTTP/1.1 204 No Content -Upload-Offset: 29718177 -Date: Sat, 24 Aug 2019 00:43:18 GMT - -Update image description - PUT - -# let's change the tag in description from 1 to latest -# once the change is made in sample.json (or your json file) - -Request - -curl -i -F "metadata= /tmp/sample_image - This is a dummy file for testing. -EOF -fi - -IMAGE_SIZE=$(ls -al /tmp/sample_image | awk '{print $5}') - -if true ; then - cat <<- EOF > /tmp/sample.json - { - "owner": "alpha", - "cluster_name": "beta", - "type": "container", - "image_name": "qwerty123", - "image_length": $IMAGE_SIZE, - "image_offset": 0, - "upload_complete": false, - "description": { - "image_records": [ - { - "image_record_name": "iuysdi1234", - "repo": "icn", - "tag": "2" - } - ] - } - } -EOF -fi - -cur_status="" - -while [[ $cur_status != "Running" ]]; do - - cur_status=$(kubectl get pods | grep bpa-api-deployment | awk '{print $3}') - if [[ $cur_status != "Running" ]]; then - echo "$(date +%H:%M:%S) - BPA-RESTful-API Pod status: $cur_status" - else - echo "$(date +%H:%M:%S) - BPA-RESTful-API Pod status: $cur_status" - break - - fi - if [[ $cur_status == "Err"* ]]; then - exit 1 - fi - sleep 10 -done - -sleep 30 - -#Get CLusterIP -IP=$(kubectl get services | grep bpa-api-service | awk '{print $3}') - -call_api -i -F "metadata= /dev/null; then - sudo apt-get -yq install golang-go -fi diff --git a/cmd/bpa-restapi-agent/internal/app/image.go b/cmd/bpa-restapi-agent/internal/app/image.go deleted file mode 100644 index 6b58100..0000000 --- a/cmd/bpa-restapi-agent/internal/app/image.go +++ /dev/null @@ -1,310 +0,0 @@ -package app - -import ( - "bpa-restapi-agent/internal/db" - "encoding/json" - "os" - "os/user" - "path" - - pkgerrors "github.com/pkg/errors" -) - -// Image contains the parameters needed for Image information -type Image struct { - Owner string `json:"owner"` - ClusterName string `json:"cluster_name"` - Type string `json:"type"` - ImageName string `json:"image_name"` - ImageOffset *int `json:"image_offset"` - ImageLength int `json:"image_length"` - UploadComplete *bool `json:"upload_complete"` - Description ImageRecordList `json:"description"` -} - -type ImageRecordList struct { - ImageRecords []map[string]string `json:"image_records"` -} - -// ImageKey is the key structure that is used in the database -type ImageKey struct { - ImageName string `json:"image_name"` -} - -// We will use json marshalling to convert to string to -// preserve the underlying structure. -func (dk ImageKey) String() string { - out, err := json.Marshal(dk) - if err != nil { - return "" - } - - return string(out) -} - -// ImageManager is an interface that exposes the Image functionality -type ImageManager interface { - Create(c Image) (Image, error) - Get(imageName string) (Image, error) - Delete(imageName string) error - Update(imageName string, c Image) (Image, error) - GetImageRecordByName(imgname, imageName string) (map[string]string, error) - GetDirPath(imageName string) (string, string, error) -} - -// Interface to aid unit test by mocking third party packages -type Utility interface { - GetCurrentUser() (*user.User, error) - DBCreate(storeName string, key ImageKey, meta string, c Image) error - DBRead(storeName string, key ImageKey, meta string) ([]byte, error) - DBUnmarshal(value []byte) (Image, error) - GetPath(user *user.User, imageName string, storeName string) (string, string) - DBDelete(storeName string, key ImageKey, meta string) error - DBUpdate(storeName string, key ImageKey, tagMeta string, c Image) error -} - -// ImageClient implements the ImageManager -// It will also be used to maintain some localized state -type ImageClient struct { - util Utility - storeName string - tagMeta string -} - -type DBService struct { -} - -// To Do - Fix repetition in -// NewImageClient returns an instance of the ImageClient -// which implements the ImageManager -func NewBinaryImageClient() *ImageClient { - service := DBService{} - return &ImageClient{ - util: service, - storeName: "binary_images", - tagMeta: "metadata", - } -} - -func NewContainerImageClient() *ImageClient { - service := DBService{} - return &ImageClient{ - util: service, - storeName: "container_images", - tagMeta: "metadata", - } -} - -func NewOSImageClient() *ImageClient { - service := DBService{} - return &ImageClient{ - util: service, - storeName: "os_images", - tagMeta: "metadata", - } -} - -// Create an entry for the Image resource in the database` -func (v *ImageClient) Create(c Image) (Image, error) { - //Construct composite key consisting of name - key := ImageKey{ - ImageName: c.ImageName, - } - - //Check if this Image already exists - _, err := v.Get(c.ImageName) - if err == nil { - return Image{}, pkgerrors.New("Image already exists") - } - - err = v.util.DBCreate(v.storeName, key, v.tagMeta, c) - if err != nil { - return Image{}, pkgerrors.Wrap(err, "Creating DB Entry") - } - - err = v.CreateFile(c) - if err != nil { - return Image{}, pkgerrors.Wrap(err, "Creating File in FS") - } - - return c, nil -} - -func (d DBService) DBCreate(storeName string, key ImageKey, meta string, c Image) error { - - //Construct composite key consisting of name - err := db.DBconn.Create(storeName, key, meta, c) - if err != nil { - return pkgerrors.Wrap(err, "Creating DB Entry") - } - - return nil -} - -// Create file - -func (v *ImageClient) CreateFile(c Image) error { - filePath, dirPath, err := v.GetDirPath(c.ImageName) - if err != nil { - return pkgerrors.Wrap(err, "Get file path") - } - err = os.MkdirAll(dirPath, 0744) - if err != nil { - return pkgerrors.Wrap(err, "Make image directory") - } - file, err := os.Create(filePath) - if err != nil { - return pkgerrors.Wrap(err, "Create image file") - } - defer file.Close() - - return nil -} - -// Get returns Image for corresponding to name -func (v *ImageClient) Get(imageName string) (Image, error) { - - //Construct the composite key to select the entry - key := ImageKey{ - // Owner: ownerName, - // ClusterName: clusterName, - ImageName: imageName, - } - - value, err := v.util.DBRead(v.storeName, key, v.tagMeta) - if err != nil { - return Image{}, pkgerrors.Wrap(err, "Get Image") - } - - //value is a byte array - if value != nil { - c, err := v.util.DBUnmarshal(value) - if err != nil { - return Image{}, pkgerrors.Wrap(err, "Unmarshaling Value") - } - return c, nil - } - - return Image{}, pkgerrors.New("Error getting Connection") -} - -func (d DBService) DBRead(storeName string, key ImageKey, meta string) ([]byte, error) { - value, err := db.DBconn.Read(storeName, key, meta) - if err != nil { - return []byte{}, pkgerrors.Wrap(err, "Get Image") - } - - return value, nil -} - -func (d DBService) DBUnmarshal(value []byte) (Image, error) { - c := Image{} - err := db.DBconn.Unmarshal(value, &c) - if err != nil { - return Image{}, pkgerrors.Wrap(err, "Unmarshaling Value") - } - - return c, nil -} - -func (v *ImageClient) GetImageRecordByName(imgName string, - imageRecordName string) (map[string]string, error) { - - img, err := v.Get(imgName) - if err != nil { - return nil, pkgerrors.Wrap(err, "Error getting image") - } - - for _, value := range img.Description.ImageRecords { - if imageRecordName == value["image_record_name"] { - return value, nil - } - } - - return nil, pkgerrors.New("Image record " + imageRecordName + " not found") -} - -func (v *ImageClient) GetDirPath(imageName string) (string, string, error) { - u, err := v.util.GetCurrentUser() - filePath, dirPath := v.util.GetPath(u, imageName, v.storeName) - - return filePath, dirPath, err -} - -// Delete the Image from database -func (v *ImageClient) Delete(imageName string) error { - - //Construct the composite key to select the entry - key := ImageKey{ - // Owner: ownerName, - // ClusterName: clusterName, - ImageName: imageName, - } - err := v.util.DBDelete(v.storeName, key, v.tagMeta) - - //Delete image from FS - filePath, _, err := v.GetDirPath(imageName) - if err != nil { - return pkgerrors.Wrap(err, "Get file path") - } - err = os.Remove(filePath) - if err != nil { - return pkgerrors.Wrap(err, "Delete image file") - } - - return nil -} - -func (d DBService) DBDelete(storeName string, key ImageKey, tagMeta string) error { - err := db.DBconn.Delete(storeName, key, tagMeta) - if err != nil { - return pkgerrors.Wrap(err, "Delete Image") - } - - return nil -} - -// Update an entry for the image in the database -func (v *ImageClient) Update(imageName string, c Image) (Image, error) { - - key := ImageKey{ - ImageName: imageName, - } - - //Check if this Image exists - _, err := v.Get(imageName) - if err != nil { - return Image{}, pkgerrors.New("Update Error - Image doesn't exist") - } - - err = v.util.DBUpdate(v.storeName, key, v.tagMeta, c) - - return c, nil -} - -func (d DBService) DBUpdate(storeName string, key ImageKey, tagMeta string, c Image) error { - err := db.DBconn.Update(storeName, key, tagMeta, c) - if err != nil { - return pkgerrors.Wrap(err, "Updating DB Entry") - } - - return nil -} - -// Define GetCurrentUser -func (d DBService) GetCurrentUser() (*user.User, error) { - u, err := user.Current() - if err != nil { - return nil, pkgerrors.Wrap(err, "Current user") - } - - return u, nil -} - -func (d DBService) GetPath(user *user.User, imageName string, storeName string) (string, string) { - home := user.HomeDir - dirPath := path.Join(home, "images", storeName) - filePath := path.Join(dirPath, imageName) - - return filePath, dirPath -} diff --git a/cmd/bpa-restapi-agent/internal/app/image_test.go b/cmd/bpa-restapi-agent/internal/app/image_test.go deleted file mode 100644 index e609477..0000000 --- a/cmd/bpa-restapi-agent/internal/app/image_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package app - -import ( - "fmt" - "io/ioutil" - "log" - "os" - "os/user" - "path" - "testing" - - "github.com/stretchr/testify/mock" - "github.com/pkg/errors" - -) - -type mockValues struct { - mock.Mock -} - -func (m *mockValues) GetCurrentUser() (*user.User, error) { - fmt.Println("Mocked Get User") - args := m.Called() - - return args.Get(0).(*user.User), args.Error(1) -} - -func TestGetDirPath(t *testing.T) { - fakeUser := user.User{} - u := &fakeUser - - myMocks := new(mockValues) - - myMocks.On("GetCurrentUser").Return(&fakeUser, nil) - myMocks.On("GetPath", u, "", "test_image").Return("", "") - - imageClient := ImageClient{myMocks, "test_image", "test_meta"} - _, dir, err := imageClient.GetDirPath("") - if err != nil { - t.Errorf("Path was incorrect, got: %q, want: %q.", dir, "some_path") - } -} - -func (m *mockValues) DBCreate(name string, key ImageKey, meta string, c Image) error{ - fmt.Println("Mocked Create image in Mongo") - args := m.Called(name, key, meta, c) - - return args.Error(0) -} - -func (m *mockValues) DBRead(name string, key ImageKey, meta string) ([]byte, error) { - fmt.Println("Mocked Mongo DB Read Operations") - args := m.Called(name, key, meta) - - return args.Get(0).([]byte), args.Error(1) -} - -func (m *mockValues) DBUnmarshal(value []byte) (Image, error) { - fmt.Println("Mocked Mongo DB Unmarshal Operation") - args := m.Called(value) - - return args.Get(0).(Image), args.Error(1) -} - -func (m *mockValues) GetPath(u *user.User, imageName string, storeName string) (string, string) { - args := m.Called(u, "", "test_image") - - return args.String(0), args.String(1) -} - -func (m *mockValues) DBDelete(name string, key ImageKey, meta string) error { - fmt.Println("Mocked Mongo DB Delete") - args := m.Called(name, key, meta) - - return args.Error(0) - -} - -func (m *mockValues) DBUpdate(s string, k ImageKey, t string, c Image) error { - fmt.Println("Mocked Mongo DB Update") - args := m.Called(s, k, t, c) - - return args.Error(0) -} - -func TestCreate(t *testing.T) { - dir, err := ioutil.TempDir("", "test_images") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(dir) - - image := Image{ - ImageName: "test_asdf", - } - arr_data := []byte{} - key := ImageKey{ImageName:"test_asdf"} - myMocks := new(mockValues) - // just to get an error value - err1 := errors.New("math: square root of negative number") - - fakeUser := user.User{} - u := &fakeUser - file_path := path.Join(dir, "test_asdf") - - myMocks.On("DBCreate", "test_image", key, "test_meta", image).Return(nil) - myMocks.On("DBRead", "test_image", key, "test_meta").Return(arr_data, err1) - myMocks.On("DBUnmarshal", arr_data).Return(image, nil) - myMocks.On("GetCurrentUser").Return(&fakeUser, nil) - myMocks.On("GetPath", u, "", "test_image").Return(file_path, dir) - - imageClient := ImageClient{myMocks, "test_image", "test_meta"} - _, err = imageClient.Create(image) - if err != nil { - t.Errorf("Some error occured!") - } -} - -func TestDelete(t *testing.T) { - tmpfile, err := ioutil.TempFile("", "test_images") - if err != nil { - log.Fatal(err) - } - defer os.Remove(tmpfile.Name()) - - key := ImageKey{ImageName: "test_asdf"} - fakeUser := user.User{} - u := &fakeUser - file_path := tmpfile.Name() - - myMocks := new(mockValues) - - myMocks.On("DBDelete", "test_image", key, "test_meta").Return(nil) - myMocks.On("GetCurrentUser").Return(&fakeUser, nil) - myMocks.On("GetPath", u, "", "test_image").Return(file_path, "") - - imageClient := ImageClient{myMocks, "test_image", "test_meta"} - err = imageClient.Delete("test_asdf") - if err != nil { - t.Errorf("Some error occured!") - } - -} - -func TestUpdate(t *testing.T) { - image := Image{} - key := ImageKey{} - arr_data := []byte{} - - myMocks := new(mockValues) - - myMocks.On("DBRead", "test_image", key, "test_meta").Return(arr_data, nil) - myMocks.On("DBUnmarshal", arr_data).Return(image, nil) - myMocks.On("DBUpdate", "test_image", key, "test_meta", image).Return(nil) - imageClient := ImageClient{myMocks, "test_image", "test_meta"} - _, err := imageClient.Update("", image) - if err != nil { - t.Errorf("Some error occured!") - } -} diff --git a/cmd/bpa-restapi-agent/internal/auth/auth.go b/cmd/bpa-restapi-agent/internal/auth/auth.go deleted file mode 100644 index 3da8f2a..0000000 --- a/cmd/bpa-restapi-agent/internal/auth/auth.go +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2018 Intel Corporation, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package auth - -import ( - "crypto/tls" - "crypto/x509" - "encoding/base64" - "encoding/pem" - "io/ioutil" - "log" - - pkgerrors "github.com/pkg/errors" -) - -// GetTLSConfig initializes a tlsConfig using the CA's certificate -// This config is then used to enable the server for mutual TLS -func GetTLSConfig(caCertFile string, certFile string, keyFile string) (*tls.Config, error) { - - // Initialize tlsConfig once - caCert, err := ioutil.ReadFile(caCertFile) - - if err != nil { - return nil, pkgerrors.Wrap(err, "Read CA Cert file") - } - - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - tlsConfig := &tls.Config{ - // Change to RequireAndVerify once we have mandatory certs - ClientAuth: tls.VerifyClientCertIfGiven, - ClientCAs: caCertPool, - MinVersion: tls.VersionTLS12, - } - - certPEMBlk, err := readPEMBlock(certFile) - if err != nil { - return nil, pkgerrors.Wrap(err, "Read Cert File") - } - - keyPEMBlk, err := readPEMBlock(keyFile) - if err != nil { - return nil, pkgerrors.Wrap(err, "Read Key File") - } - - tlsConfig.Certificates = make([]tls.Certificate, 1) - tlsConfig.Certificates[0], err = tls.X509KeyPair(certPEMBlk, keyPEMBlk) - if err != nil { - return nil, pkgerrors.Wrap(err, "Load x509 cert and key") - } - - tlsConfig.BuildNameToCertificate() - return tlsConfig, nil -} - -func readPEMBlock(filename string) ([]byte, error) { - - pemData, err := ioutil.ReadFile(filename) - if err != nil { - return nil, pkgerrors.Wrap(err, "Read PEM File") - } - - pemBlock, rest := pem.Decode(pemData) - if len(rest) > 0 { - log.Println("Pemfile has extra data") - } - - if x509.IsEncryptedPEMBlock(pemBlock) { - password, err := ioutil.ReadFile(filename + ".pass") - if err != nil { - return nil, pkgerrors.Wrap(err, "Read Password File") - } - - pByte, err := base64.StdEncoding.DecodeString(string(password)) - if err != nil { - return nil, pkgerrors.Wrap(err, "Decode PEM Password") - } - - pemData, err = x509.DecryptPEMBlock(pemBlock, pByte) - if err != nil { - return nil, pkgerrors.Wrap(err, "Decrypt PEM Data") - } - var newPEMBlock pem.Block - newPEMBlock.Type = pemBlock.Type - newPEMBlock.Bytes = pemData - // Converting back to PEM from DER data you get from - // DecryptPEMBlock - pemData = pem.EncodeToMemory(&newPEMBlock) - } - - return pemData, nil -} diff --git a/cmd/bpa-restapi-agent/internal/config/config.go b/cmd/bpa-restapi-agent/internal/config/config.go deleted file mode 100644 index e4911d2..0000000 --- a/cmd/bpa-restapi-agent/internal/config/config.go +++ /dev/null @@ -1,64 +0,0 @@ -package config - -import ( - "encoding/json" - "log" - "os" -) - -type Configuration struct { - Password string `json: "password"` - DatabaseAddress string `json: "database-address"` - DatabaseType string `json: "database-type"` - ServicePort string `json: "service-port"` - MinIOAddress string `json: "minio-address"` - MinIOPort string `json: "minio-port"` - AccessKeyID string `json: "access-key-id"` - SecretAccessKey string `json: "secret-access-key"` -} - -var gConfig *Configuration - -func readConfigFile(file string) (*Configuration, error) { - f, err := os.Open(file) - if err != nil { - return defaultConfiguration(), err - } - defer f.Close() - - conf := defaultConfiguration() - - decoder := json.NewDecoder(f) - err = decoder.Decode(conf) - if err != nil { - return conf, err - } - - return conf, nil -} - -func defaultConfiguration() *Configuration { - return &Configuration { - Password: "", - DatabaseAddress: "127.0.0.1", - DatabaseType: "mongo", - ServicePort: "9015", - MinIOAddress: "127.0.0.1", - MinIOPort: "9000", - AccessKeyID: "ICN-ACCESSKEYID", - SecretAccessKey: "ICN-SECRETACCESSKEY", - } -} - -func GetConfiguration() *Configuration { - if gConfig == nil { - conf, err := readConfigFile("ICNconfig.json") - if err != nil { - log.Println("Error loading config file. Using defaults") - } - - gConfig = conf - } - - return gConfig -} diff --git a/cmd/bpa-restapi-agent/internal/db/mongo.go b/cmd/bpa-restapi-agent/internal/db/mongo.go deleted file mode 100644 index 454f26c..0000000 --- a/cmd/bpa-restapi-agent/internal/db/mongo.go +++ /dev/null @@ -1,379 +0,0 @@ -package db - -import ( - "golang.org/x/net/context" - "log" - - "bpa-restapi-agent/internal/config" - - pkgerrors "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -// MongoCollection defines the a subset of MongoDB operations -// Note: This interface is defined mainly for mock testing -type MongoCollection interface { - InsertOne(ctx context.Context, document interface{}, - opts ...*options.InsertOneOptions) (*mongo.InsertOneResult, error) - FindOne(ctx context.Context, filter interface{}, - opts ...*options.FindOneOptions) *mongo.SingleResult - FindOneAndUpdate(ctx context.Context, filter interface{}, - update interface{}, opts ...*options.FindOneAndUpdateOptions) *mongo.SingleResult - DeleteOne(ctx context.Context, filter interface{}, - opts ...*options.DeleteOptions) (*mongo.DeleteResult, error) - Find(ctx context.Context, filter interface{}, - opts ...*options.FindOptions) (*mongo.Cursor, error) -} - -// MongoStore is an implementation of the db.Store interface -type MongoStore struct { - db *mongo.Database -} - -// This exists only for allowing us to mock the collection object -// for testing purposes -var getCollection = func(coll string, m *MongoStore) MongoCollection { - return m.db.Collection(coll) -} - -// This exists only for allowing us to mock the DecodeBytes function -// Mainly because we cannot construct a SingleResult struct from our -// tests. All fields in that struct are private. -var decodeBytes = func(sr *mongo.SingleResult) (bson.Raw, error) { - return sr.DecodeBytes() -} - -// These exists only for allowing us to mock the cursor.Next function -// Mainly because we cannot construct a mongo.Cursor struct from our -// tests. All fields in that struct are private and there is no public -// constructor method. -var cursorNext = func(ctx context.Context, cursor *mongo.Cursor) bool { - return cursor.Next(ctx) -} -var cursorClose = func(ctx context.Context, cursor *mongo.Cursor) error { - return cursor.Close(ctx) -} - -// NewMongoStore initializes a Mongo Database with the name provided -// If a database with that name exists, it will be returned -func NewMongoStore(name string, store *mongo.Database) (Store, error) { - if store == nil { - ip := "mongodb://" + config.GetConfiguration().DatabaseAddress + ":27017" - clientOptions := options.Client() - clientOptions.ApplyURI(ip) - mongoClient, err := mongo.NewClient(clientOptions) - if err != nil { - return nil, err - } - - err = mongoClient.Connect(context.Background()) - if err != nil { - return nil, err - } - store = mongoClient.Database(name) - } - - return &MongoStore{ - db: store, - }, nil -} - -// HealthCheck verifies if the database is up and running -func (m *MongoStore) HealthCheck() error { - - _, err := decodeBytes(m.db.RunCommand(context.Background(), bson.D{{"serverStatus", 1}})) - if err != nil { - return pkgerrors.Wrap(err, "Error getting server status") - } - - return nil -} - -// validateParams checks to see if any parameters are empty -func (m *MongoStore) validateParams(args ...interface{}) bool { - for _, v := range args { - val, ok := v.(string) - if ok { - if val == "" { - return false - } - } else { - if v == nil { - return false - } - } - } - - return true -} - -// Create is used to create a DB entry -func (m *MongoStore) Create(coll string, key Key, tag string, data interface{}) error { - if data == nil || !m.validateParams(coll, key, tag) { - return pkgerrors.New("No Data to store") - } - - c := getCollection(coll, m) - ctx := context.Background() - - //Insert the data and then add the objectID to the masterTable - res, err := c.InsertOne(ctx, bson.D{ - {tag, data}, - }) - if err != nil { - return pkgerrors.Errorf("Error inserting into database: %s", err.Error()) - } - - //Add objectID of created data to masterKey document - //Create masterkey document if it does not exist - filter := bson.D{{"key", key}} - - _, err = decodeBytes( - c.FindOneAndUpdate( - ctx, - filter, - bson.D{ - {"$set", bson.D{ - {tag, res.InsertedID}, - }}, - }, - options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After))) - - if err != nil { - return pkgerrors.Errorf("Error updating master table: %s", err.Error()) - } - - return nil -} - -// Update is used to update a DB entry -func (m *MongoStore) Update(coll string, key Key, tag string, data interface{}) error { - if data == nil || !m.validateParams(coll, key, tag) { - return pkgerrors.New("No Data to update") - } - - c := getCollection(coll, m) - ctx := context.Background() - - //Get the masterkey document based on given key - filter := bson.D{{"key", key}} - keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) - if err != nil { - return pkgerrors.Errorf("Error finding master table: %s", err.Error()) - } - - //Read the tag objectID from document - tagoid, ok := keydata.Lookup(tag).ObjectIDOK() - if !ok { - return pkgerrors.Errorf("Error finding objectID for tag %s", tag) - } - - //Update the document with new data - filter = bson.D{{"_id", tagoid}} - - _, err = decodeBytes( - c.FindOneAndUpdate( - ctx, - filter, - bson.D{ - {"$set", bson.D{ - {tag, data}, - }}, - }, - options.FindOneAndUpdate().SetReturnDocument(options.After))) - - if err != nil { - return pkgerrors.Errorf("Error updating record: %s", err.Error()) - } - - return nil -} - -// Unmarshal implements an unmarshaler for bson data that -// is produced from the mongo database -func (m *MongoStore) Unmarshal(inp []byte, out interface{}) error { - err := bson.Unmarshal(inp, out) - if err != nil { - return pkgerrors.Wrap(err, "Unmarshaling bson") - } - return nil -} - -// Read method returns the data stored for this key and for this particular tag -func (m *MongoStore) Read(coll string, key Key, tag string) ([]byte, error) { - if !m.validateParams(coll, key, tag) { - return nil, pkgerrors.New("Mandatory fields are missing") - } - - c := getCollection(coll, m) - ctx := context.Background() - - //Get the masterkey document based on given key - filter := bson.D{{"key", key}} - keydata, err := decodeBytes(c.FindOne(context.Background(), filter)) - if err != nil { - return nil, pkgerrors.Errorf("Error finding master table: %s", err.Error()) - } - - //Read the tag objectID from document - tagoid, ok := keydata.Lookup(tag).ObjectIDOK() - if !ok { - return nil, pkgerrors.Errorf("Error finding objectID for tag %s", tag) - } - - //Use tag objectID to read the data from store - filter = bson.D{{"_id", tagoid}} - tagdata, err := decodeBytes(c.FindOne(ctx, filter)) - if err != nil { - return nil, pkgerrors.Errorf("Error reading found object: %s", err.Error()) - } - - //Return the data as a byte array - //Convert string data to byte array using the built-in functions - switch tagdata.Lookup(tag).Type { - case bson.TypeString: - return []byte(tagdata.Lookup(tag).StringValue()), nil - default: - return tagdata.Lookup(tag).Value, nil - } -} - -// Helper function that deletes an object by its ID -func (m *MongoStore) deleteObjectByID(coll string, objID primitive.ObjectID) error { - - c := getCollection(coll, m) - ctx := context.Background() - - _, err := c.DeleteOne(ctx, bson.D{{"_id", objID}}) - if err != nil { - return pkgerrors.Errorf("Error Deleting from database: %s", err.Error()) - } - - log.Printf("Deleted Obj with ID %s", objID.String()) - return nil -} - -// Delete method removes a document from the Database that matches key -// TODO: delete all referenced docs if tag is empty string -func (m *MongoStore) Delete(coll string, key Key, tag string) error { - if !m.validateParams(coll, key, tag) { - return pkgerrors.New("Mandatory fields are missing") - } - - c := getCollection(coll, m) - ctx := context.Background() - - //Get the masterkey document based on given key - filter := bson.D{{"key", key}} - //Remove the tag ID entry from masterkey table - update := bson.D{ - { - "$unset", bson.D{ - {tag, ""}, - }, - }, - } - keydata, err := decodeBytes(c.FindOneAndUpdate(ctx, filter, update, - options.FindOneAndUpdate().SetReturnDocument(options.Before))) - if err != nil { - //No document was found. Return nil. - if err == mongo.ErrNoDocuments { - return nil - } - //Return any other error that was found. - return pkgerrors.Errorf("Error decoding master table after update: %s", - err.Error()) - } - - //Read the tag objectID from document - elems, err := keydata.Elements() - if err != nil { - return pkgerrors.Errorf("Error reading elements from database: %s", err.Error()) - } - - tagoid, ok := keydata.Lookup(tag).ObjectIDOK() - if !ok { - return pkgerrors.Errorf("Error finding objectID for tag %s", tag) - } - - //Use tag objectID to read the data from store - err = m.deleteObjectByID(coll, tagoid) - if err != nil { - return pkgerrors.Errorf("Error deleting from database: %s", err.Error()) - } - - //Delete master table if no more tags left - //_id, key and tag should be elements in before doc - //if master table needs to be removed too - if len(elems) == 3 { - keyid, ok := keydata.Lookup("_id").ObjectIDOK() - if !ok { - return pkgerrors.Errorf("Error finding objectID for key %s", key) - } - err = m.deleteObjectByID(coll, keyid) - if err != nil { - return pkgerrors.Errorf("Error deleting master table from database: %s", err.Error()) - } - } - - return nil -} - -// ReadAll is used to get all documents in db of a particular tag -func (m *MongoStore) ReadAll(coll, tag string) (map[string][]byte, error) { - if !m.validateParams(coll, tag) { - return nil, pkgerrors.New("Missing collection or tag name") - } - - c := getCollection(coll, m) - ctx := context.Background() - - //Get all master tables in this collection - filter := bson.D{ - {"key", bson.D{ - {"$exists", true}, - }}, - } - cursor, err := c.Find(ctx, filter) - if err != nil { - return nil, pkgerrors.Errorf("Error reading from database: %s", err.Error()) - } - defer cursorClose(ctx, cursor) - - //Iterate over all the master tables - result := make(map[string][]byte) - for cursorNext(ctx, cursor) { - d := cursor.Current - - //Read key of each master table - key, ok := d.Lookup("key").DocumentOK() - if !ok { - //Throw error if key is not found - pkgerrors.New("Unable to read key from mastertable") - } - - //Get objectID of tag document - tid, ok := d.Lookup(tag).ObjectIDOK() - if !ok { - log.Printf("Did not find tag: %s", tag) - continue - } - - //Find tag document and unmarshal it into []byte - tagData, err := decodeBytes(c.FindOne(ctx, bson.D{{"_id", tid}})) - if err != nil { - log.Printf("Unable to decode tag data %s", err.Error()) - continue - } - result[key.String()] = tagData.Lookup(tag).Value - } - - if len(result) == 0 { - return result, pkgerrors.Errorf("Did not find any objects with tag: %s", tag) - } - - return result, nil -} diff --git a/cmd/bpa-restapi-agent/internal/db/store.go b/cmd/bpa-restapi-agent/internal/db/store.go deleted file mode 100644 index 0b981e7..0000000 --- a/cmd/bpa-restapi-agent/internal/db/store.go +++ /dev/null @@ -1,75 +0,0 @@ -package db - -import ( - "encoding/json" - "reflect" - - pkgerrors "github.com/pkg/errors" -) - -// DBconn interface used to talk to a concrete Database connection -var DBconn Store - -// Key is an interface that will be implemented by anypackage -// that wants to use the Store interface. This allows various -// db backends and key types. -type Key interface { - String() string -} - -// Store is an interface for accessing a database -type Store interface { - // Returns nil if db health is good - HealthCheck() error - - // Unmarshal implements any unmarshaling needed for the database - Unmarshal(inp []byte, out interface{}) error - - // Creates a new master table with key and links data with tag and - // creates a pointer to the newly added data in the master table - Create(table string, key Key, tag string, data interface{}) error - - // Reads data for a particular key with specific tag. - Read(table string, key Key, tag string) ([]byte, error) - - // Update data for particular key with specific tag - Update(table string, key Key, tag string, data interface{}) error - - // Deletes a specific tag data for key. - // TODO: If tag is empty, it will delete all tags under key. - Delete(table string, key Key, tag string) error - - // Reads all master tables and data from the specified tag in table - ReadAll(table string, tag string) (map[string][]byte, error) -} - -// CreateDBClient creates the DB client -func CreateDBClient(dbType string) error { - var err error - switch dbType { - case "mongo": - // create a mongodb database with ICN as the name - DBconn, err = NewMongoStore("icn", nil) - default: - return pkgerrors.New(dbType + "DB not supported") - } - return err -} - -// Serialize converts given data into a JSON string -func Serialize(v interface{}) (string, error) { - out, err := json.Marshal(v) - if err != nil { - return "", pkgerrors.Wrap(err, "Error serializing "+reflect.TypeOf(v).String()) - } - return string(out), nil -} - -// DeSerialize converts string to a json object specified by type -func DeSerialize(str string, v interface{}) error { - err := json.Unmarshal([]byte(str), &v) - if err != nil { - return pkgerrors.Wrap(err, "Error deSerializing "+str) - } - return nil -} diff --git a/cmd/bpa-restapi-agent/internal/storage/minio.go b/cmd/bpa-restapi-agent/internal/storage/minio.go deleted file mode 100644 index 3eed689..0000000 --- a/cmd/bpa-restapi-agent/internal/storage/minio.go +++ /dev/null @@ -1,183 +0,0 @@ -package storage - -import ( - "github.com/minio/minio-go/v6" - "bpa-restapi-agent/internal/config" - - "log" - "os" -) - -type MinIOInfo struct { - minioC *minio.Client `json:"minio client"` -} - -// Initialize the MinIO server, create buckets -func Initialize() (MinIOInfo, error) { - endpoint := config.GetConfiguration().MinIOAddress + ":" + config.GetConfiguration().MinIOPort - accessKeyID := config.GetConfiguration().AccessKeyID - secretAccessKey := config.GetConfiguration().SecretAccessKey - useSSL := false - - minioInfo := MinIOInfo{} - // Initialize minio client object. - minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) - if err != nil { - log.Fatalln(err) - return minioInfo, err - } - - // Make a new bucket. - bucketNames := []string{"binary", "container", "operatingsystem"} - location := "us-west-1" - - for _, bucketName := range bucketNames { - err := minioClient.MakeBucket(bucketName, location) - if err != nil { - // Check to see if we already own this bucket (which happens if you run this twice) - exists, errBucketExists := minioClient.BucketExists(bucketName) - if errBucketExists == nil && exists { - log.Printf("We already own %s\n", bucketName) - } else { - log.Fatalln(err) - return minioInfo, err - } - } else { - log.Printf("Successfully created %s\n", bucketName) - } - } - - minioInfo.minioC = minioClient - return minioInfo, nil -} - -func (m MinIOInfo) PutImage(bucketName string, objName string, localPath string) (int64, error) { - - //contentType := "multipart/form-data" - contentType := "application/octet-stream" - - // Upload the zip file with FPutObject - n, err := m.minioC.FPutObject(bucketName, objName, localPath, minio.PutObjectOptions{ContentType:contentType}) - if err != nil { - log.Fatalln(err) - return n, err - } - - fileInfo, _ := os.Stat(localPath) - fileSize := fileInfo.Size() - - if n != int64(fileSize) { - log.Printf("FPutObject failed %s of size %d\n", objName, n) - return n, err - } - - log.Printf("Successfully uploaded %s of size %d\n", objName, n) - return n, nil -} - -func (m MinIOInfo) PatchImage(bucketName string, objName string, localPath string, offset int64, objSize int64) (int64, error) { - - var n = int64(0) - - tempFile, err := os.Open(localPath) - if err != nil { - log.Fatalln(err) - return n, err - } - - defer tempFile.Close() - - if _, err := tempFile.Seek(offset, 0); err != nil { - log.Printf("PatchImage seek %s failed: %s", tempFile.Name(), err) - return n, err - } - - objInfo, err := m.minioC.StatObject(bucketName, objName, minio.StatObjectOptions{}) - var objHealthy = true - if err != nil { - objHealthy = false - } else if objInfo.Size != offset || objInfo.Size == 0 { - objHealthy = false - } - - var objNameTemp = objName - if objHealthy { - objNameTemp = objName + ".tmp" - } - - contentType := "application/octet-stream" - n, err = m.minioC.PutObject(bucketName, objNameTemp, tempFile, objSize, minio.PutObjectOptions{ContentType:contentType}) - if err != nil { - log.Fatalln(err) - return n, err - } - - if n != objSize { - log.Printf("PatchImage PutObject %s failed with bytes: %d", tempFile.Name(), n) - return n, err - } - - if objHealthy { - src1 := minio.NewSourceInfo(bucketName, objName, nil) - src2 := minio.NewSourceInfo(bucketName, objNameTemp, nil) - srcs := []minio.SourceInfo{src1, src2} - - dst, err := minio.NewDestinationInfo(bucketName, objName, nil, nil) - if err != nil { - log.Printf("NewDestinationInfo failed", err) - return n, err - } - - // There is issue, the last src should be the smallest obj size - err = m.minioC.ComposeObject(dst, srcs) - if err != nil { - log.Printf("ComposeObject failed", err) - return n, err - } - } - - log.Printf("Successfully PatchImage %s of size %d\n", objName, n) - return n, nil -} - -func (m MinIOInfo) DeleteImage(bucketName string, objName string) (error) { - - err := m.minioC.RemoveObject(bucketName, objName) - if err != nil { - log.Printf("MinIO Remove object %s failed\n", bucketName) - return err - } - - return nil -} - -func (m MinIOInfo) CleanupImages(bucketName string) (error) { - // create a done channel to control 'ListObjectsV2' go routine. - doneCh := make(chan struct{}) - defer close(doneCh) - - for objCh := range m.minioC.ListObjectsV2(bucketName, "", true, doneCh) { - if objCh.Err != nil { - return objCh.Err - } - if objCh.Key != "" { - err := m.minioC.RemoveObject(bucketName, objCh.Key) - if err != nil { - return err - } - } - } - for objPartInfo := range m.minioC.ListIncompleteUploads(bucketName, "", true, doneCh) { - if objPartInfo.Err != nil { - return objPartInfo.Err - } - if objPartInfo.Key != "" { - err := m.minioC.RemoveIncompleteUpload(bucketName, objPartInfo.Key) - if err != nil { - return err - } - } - } - - return nil -} diff --git a/cmd/bpa-restapi-agent/internal/utils.go b/cmd/bpa-restapi-agent/internal/utils.go deleted file mode 100644 index b590789..0000000 --- a/cmd/bpa-restapi-agent/internal/utils.go +++ /dev/null @@ -1,33 +0,0 @@ -package utils - -import( - //"log" - "bpa-restapi-agent/internal/db" - "bpa-restapi-agent/internal/config" - pkgerrors "github.com/pkg/errors" -) - -func CheckDatabaseConnection() error { -// To Do - Implement db and config - - err := db.CreateDBClient(config.GetConfiguration().DatabaseType) - if err != nil { - return pkgerrors.Cause(err) - } - - err = db.DBconn.HealthCheck() - if err != nil { - return pkgerrors.Cause(err) - } - - return nil -} - -func CheckInitialSettings() error { - err := CheckDatabaseConnection() - if err != nil { - return pkgerrors.Cause(err) - } - - return nil -} diff --git a/cmd/bpa-restapi-agent/main.go b/cmd/bpa-restapi-agent/main.go deleted file mode 100644 index 26f0ba4..0000000 --- a/cmd/bpa-restapi-agent/main.go +++ /dev/null @@ -1,60 +0,0 @@ -// main.go -package main - -import ( - "context" - "log" - "math/rand" - "net/http" - "os" - "os/signal" - "time" - - - "github.com/gorilla/handlers" - - "bpa-restapi-agent/api" - utils "bpa-restapi-agent/internal" - "bpa-restapi-agent/internal/auth" - "bpa-restapi-agent/internal/config" -) - -func main() { - // check initial config - err := utils.CheckInitialSettings() - if err != nil{ - log.Fatal(err) - } - - rand.Seed(time.Now().UnixNano()) - - httpRouter := api.NewRouter(nil, nil, nil) - // Return http.handler and log requests to Stdout - loggedRouter := handlers.LoggingHandler(os.Stdout, httpRouter) - log.Println("Starting Integrated Cloud Native API") - - // Create custom http server - httpServer := &http.Server{ - Handler: loggedRouter, - Addr: ":" + config.GetConfiguration().ServicePort, - } - connectionsClose := make(chan struct{}) - go func() { - c := make(chan os.Signal, 1) // create c channel to receive notifications - signal.Notify(c, os.Interrupt) // register c channel to run concurrently - <-c - httpServer.Shutdown(context.Background()) - close(connectionsClose) - }() - - // Start server - tlsConfig, err := auth.GetTLSConfig("ca.cert", "server.cert", "server.key") - if err != nil { - log.Println("Error Getting TLS Configuration. Starting without TLS...") - log.Fatal(httpServer.ListenAndServe()) - } else { - httpServer.TLSConfig = tlsConfig - // empty strings because tlsconfig already has this information - err = httpServer.ListenAndServeTLS("", "") - } -} diff --git a/cmd/bpa-restapi-agent/service.yml b/cmd/bpa-restapi-agent/service.yml deleted file mode 100644 index b121c73..0000000 --- a/cmd/bpa-restapi-agent/service.yml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: bpa-api-service -spec: - selector: - app: bpa-api1 - ports: - - port: 9015 - targetPort: 9015 - protocol: TCP - type: NodePort ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: bpa-api-deployment -spec: - replicas: 1 - selector: - matchLabels: - app: bpa-api1 - strategy: - type: Recreate - template: - metadata: - labels: - app: bpa-api1 - spec: - serviceAccount: bpa-restapi-agent - # Refer to the PVC created earlier - volumes: - - name: storage - persistentVolumeClaim: - # Name of the PVC created earlier - claimName: minio-local-pvc - containers: - - name: bpa-api1 - image: akraino.org/icn/bpa-restapi-agent:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9015 - - name: mongo - image: mongo - ports: - - containerPort: 27017 - - name: minio - # Pulls the default Minio image from Docker Hub - image: minio/minio:latest - args: - - server - - /storage - env: - # Minio access key and secret key - - name: MINIO_ACCESS_KEY - value: "ICN-ACCESSKEYID" - - name: MINIO_SECRET_KEY - value: "ICN-SECRETACCESSKEY" - ports: - - containerPort: 9000 - # Mount the volume into the pod - volumeMounts: - - name: storage # must match the volume name, above - mountPath: "/storage"