Add fetch requirements from site

This entrypoint will fetch requirements from a site, and
will store on $HOME/.kni/$SITE_NAME. It accepts a single
argument, the path/url to a site (in go-getter format)

Signed-off-by: Yolanda Robla <yroblamo@redhat.com>
Change-Id: Id9025fa7d14e90a8f03ee07d1e550b48d0003ae6

diff --git a/cmd/fetch_requirements.go b/cmd/fetch_requirements.go
new file mode 100644 (file)
index 0000000..a898bd0
--- /dev/null
+++ b/cmd/fetch_requirements.go
@@ -0,0 +1,54 @@
+// Copyright © 2019 Red Hat <yroblamo@redhat.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+       "fmt"
+       "os"
+
+       "gerrit.akraino.org/kni/installer/pkg/site"
+       "github.com/spf13/cobra"
+)
+
+// fetchRequirementsCmd represents the fetch_requirements command
+var fetchRequirementsCmd = &cobra.Command{
+       Use:              "fetch_requirements",
+       Short:            "Command to fetch the requirements needed for a site",
+       Long:             ``,
+       TraverseChildren: true,
+       Run: func(cmd *cobra.Command, args []string) {
+               // retrieve config values and start fetching
+               siteRepo, _ := cmd.Flags().GetString("site")
+               buildPath, _ := cmd.Flags().GetString("build_path")
+               if len(buildPath) == 0 {
+                       // will generate a temporary directory
+                       buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME"))
+               }
+
+               // define a site object and proceed with requirements fetch
+               s := site.New(siteRepo, buildPath)
+               s.DownloadSite()
+               s.FetchRequirements()
+       },
+}
+
+func init() {
+       rootCmd.AddCommand(fetchRequirementsCmd)
+
+       fetchRequirementsCmd.Flags().StringP("site", "", "", "Url/path for site repository. Can be in any go-getter compatible format")
+       fetchRequirementsCmd.MarkFlagRequired("site")
+       fetchRequirementsCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that not exists, the installer will generate a default directory")
+
+}

diff --git a/pkg/requirements/requirements.go b/pkg/requirements/requirements.go
new file mode 100644 (file)
index 0000000..1fe40e8
--- /dev/null
+++ b/pkg/requirements/requirements.go
@@ -0,0 +1,122 @@
+package requirements
+
+import (
+       "bytes"
+       "fmt"
+       "io"
+       "log"
+       "os"
+       "os/exec"
+       "path"
+       "path/filepath"
+       "strings"
+
+       getter "github.com/hashicorp/go-getter"
+)
+
+// Requirement : Structure that contains the settings needed for managing a requirement
+type Requirement struct {
+       binaryName string
+       sourceRepo string
+       buildPath  string
+}
+
+// New constructor for the generator
+func New(binaryName string, sourceRepo string, buildPath string) Requirement {
+       r := Requirement{binaryName, sourceRepo, buildPath}
+       return r
+}
+
+// download requirement from a tarball or folder
+func (r Requirement) FetchRequirementFolder() {
+       // extract the tarball if exists
+       log.Println(fmt.Sprintf("Pulling %s tarball from %s", r.binaryName, r.sourceRepo))
+
+       extractDir := fmt.Sprintf("%s/%s_content", r.buildPath, r.binaryName)
+       client := &getter.Client{Src: r.sourceRepo, Dst: extractDir, Mode: getter.ClientModeAny}
+       err := client.Get()
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Error cloning tarball repository: %s", err))
+               os.Exit(1)
+       }
+
+       // find the binary inside the extracted content
+       alternativeBinaryName := path.Base(r.sourceRepo)
+       err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error {
+               if (info.Name() == r.binaryName || info.Name() == alternativeBinaryName) && !info.IsDir() {
+                       // we found the binary, move it. Give exec perms as well
+            finalName := fmt.Sprintf("%s/%s", r.buildPath, r.binaryName)
+                       os.Rename(path, finalName)
+            os.Chmod(finalName, 0755)
+                       os.RemoveAll(extractDir)
+                       return nil
+               }
+               return nil
+       })
+}
+
+// generates the openshift binary
+func (r Requirement) BuildOpenshiftBinary() {
+       extractDir := fmt.Sprintf("%s/src/github.com/openshift/installer", r.buildPath)
+       client := &getter.Client{Src: r.sourceRepo, Dst: extractDir, Mode: getter.ClientModeAny}
+       err := client.Get()
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Error cloning tarball repository: %s", err))
+               os.Exit(1)
+       }
+
+       // build the openshift binary
+       cmd := exec.Command("hack/build.sh")
+       cmd.Dir = extractDir
+       cmd.Env = os.Environ()
+       cmd.Env = append(cmd.Env, "TAGS=libvirt")
+       cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", r.buildPath))
+
+       var stdBuffer bytes.Buffer
+       mw := io.MultiWriter(os.Stdout, &stdBuffer)
+       cmd.Stdout = mw
+       cmd.Stderr = mw
+
+       err = cmd.Run()
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Error building binary: %s - %s", err, stdBuffer.String()))
+               os.Exit(1)
+       }
+       log.Println(stdBuffer.String())
+
+       // copy the generated binary to the build directory
+       cmd = exec.Command("cp", fmt.Sprintf("%s/bin/openshift-install", extractDir), r.buildPath)
+       err = cmd.Run()
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Error copying installer to buid path: %s", err))
+               os.Exit(1)
+       }
+       log.Println(fmt.Sprintf("Installer is available on %s/openshift-install", r.buildPath))
+}
+
+// download a requirement from a git repo and build it
+func (r Requirement) FetchRequirementGit() {
+       if r.binaryName == "openshift-install" {
+               r.BuildOpenshiftBinary()
+       } else {
+               log.Fatal(fmt.Sprintf("Build of binary %s is not supported", r.binaryName))
+               os.Exit(1)
+       }
+}
+
+// downloads an individual requirement
+func (r Requirement) FetchRequirement() {
+       log.Println(fmt.Sprintf("Downloading %s requirement from %s", r.binaryName, r.sourceRepo))
+
+       // first check if the binary already exists
+       binaryPath := fmt.Sprintf("%s/%s", r.buildPath, r.binaryName)
+       if _, err := os.Stat(binaryPath); err == nil {
+               log.Println(fmt.Sprintf("Using existing %s", binaryPath))
+       } else if os.IsNotExist(err) {
+               if strings.Contains(r.sourceRepo, ".git") {
+                       r.FetchRequirementGit()
+               } else {
+                       r.FetchRequirementFolder()
+               }
+       }
+}

diff --git a/pkg/site/site.go b/pkg/site/site.go
new file mode 100644 (file)
index 0000000..0da768a
--- /dev/null
+++ b/pkg/site/site.go
@@ -0,0 +1,116 @@
+package site
+
+import (
+       "bufio"
+       "fmt"
+       "io/ioutil"
+       "log"
+       "os"
+       "path"
+       "strings"
+
+       "gerrit.akraino.org/kni/installer/pkg/requirements"
+       getter "github.com/hashicorp/go-getter"
+       "gopkg.in/yaml.v2"
+)
+
+// Site : Structure that contains the settings needed for managing a site
+type Site struct {
+       siteRepo  string
+       siteName  string
+       buildPath string
+}
+
+// New constructor for the generator
+func New(siteRepo string, buildPath string) Site {
+       // given a site repo, extract the site name from the path
+       baseName := path.Base(siteRepo)
+       suffixList := [2]string{".git", ".tar.gz"}
+
+       siteName := baseName
+       for _, suffix := range suffixList {
+               siteName = strings.TrimSuffix(siteName, suffix)
+       }
+
+       s := Site{siteRepo, siteName, buildPath}
+       return s
+}
+
+// given a site repo, downloads the content and places into buildPath
+func (s Site) DownloadSite() {
+       // Clone the site repository
+       log.Println(fmt.Sprintf("Cloning the site repository from %s", s.siteRepo))
+       siteBuildPath := fmt.Sprintf("%s/%s/site", s.buildPath, s.siteName)
+       client := &getter.Client{Src: s.siteRepo, Dst: siteBuildPath, Mode: getter.ClientModeAny}
+       err := client.Get()
+       if err != nil {
+               log.Fatal(fmt.Sprintf("Error cloning site repository: %s", err))
+       }
+
+}
+
+// using the downloaded site content, fetches (and builds) the specified requirements
+func (s Site) FetchRequirements() {
+       log.Println(fmt.Sprintf("Downloading requirements for %s", s.siteName))
+       siteBuildPath := fmt.Sprintf("%s/%s", s.buildPath, s.siteName)
+
+       // searches for file containing the profile of the blueprint
+       profileFile := fmt.Sprintf("%s/site/00_install-config/kustomization.yaml", siteBuildPath)
+
+       if _, err := os.Stat(profileFile); err == nil {
+               // parse yaml and extract base
+               yamlContent, err := ioutil.ReadFile(profileFile)
+               if err != nil {
+                       log.Fatal(fmt.Sprintf("Error reading profile file: %s", err))
+                       os.Exit(1)
+               }
+
+               profileSettings := &map[string][]interface{}{}
+               err = yaml.Unmarshal(yamlContent, &profileSettings)
+               if err != nil {
+                       log.Fatal(fmt.Sprintf("Error parsing profile yaml file: %s", err))
+                       os.Exit(1)
+               }
+               bases := (*profileSettings)["bases"]
+               profileRepo := fmt.Sprintf("%s", bases[0])
+
+               // given the profile repo, we need to get the full path without file, and clone it
+               profileBits := strings.Split(profileRepo, "/")
+               profileName := profileBits[len(profileBits)-2]
+               profilePath := strings.TrimSuffix(profileRepo, profileBits[len(profileBits)-1])
+
+               profileBuildPath := fmt.Sprintf("%s/%s", siteBuildPath, profileName)
+               log.Println(fmt.Sprintf("Downloading profile repo from %s into %s", profilePath, profileBuildPath))
+               client := &getter.Client{Src: profilePath, Dst: profileBuildPath, Mode: getter.ClientModeAny}
+               err = client.Get()
+               if err != nil {
+                       log.Fatal(fmt.Sprintf("Error cloning profile repository: %s", err))
+               }
+
+               // read yaml from requirements and start fetching the bits
+               requirementsFile := fmt.Sprintf("%s/requirements.yaml", profileBuildPath)
+               file, err := os.Open(requirementsFile)
+               if err != nil {
+                       log.Fatal("Error reading requirements file")
+                       os.Exit(1)
+               }
+               defer file.Close()
+
+               scanner := bufio.NewScanner(file)
+               for scanner.Scan() {
+                       requirementsLine := scanner.Text()
+
+                       // requirements is composed of binary and source
+                       requirementsBits := strings.SplitN(strings.TrimSpace(requirementsLine), ":", 2)
+                       r := requirements.New(strings.TrimSpace(requirementsBits[0]), strings.TrimSpace(requirementsBits[1]), fmt.Sprintf("%s/requirements", siteBuildPath))
+                       r.FetchRequirement()
+               }
+
+               // remove profile folder
+               os.RemoveAll(profileBuildPath)
+
+       } else if os.IsNotExist(err) {
+               log.Fatal(fmt.Sprintf("File %s does not exist, exiting", profileFile))
+               os.Exit(1)
+       }
+}