From eb86fc6df842ce6e8043f472dd987c41b4ad4032 Mon Sep 17 00:00:00 2001 From: Yolanda Robla Date: Wed, 3 Jul 2019 17:30:21 +0200 Subject: [PATCH] Add fetch requirements from site This entrypoint will fetch requirements from a site, and will store on $HOME/.kni/$SITE_NAME. It accepts a single argument, the path/url to a site (in go-getter format) Signed-off-by: Yolanda Robla Change-Id: Id9025fa7d14e90a8f03ee07d1e550b48d0003ae6 --- cmd/fetch_requirements.go | 54 +++++++++++++++++ pkg/requirements/requirements.go | 122 +++++++++++++++++++++++++++++++++++++++ pkg/site/site.go | 116 +++++++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 cmd/fetch_requirements.go create mode 100644 pkg/requirements/requirements.go create mode 100644 pkg/site/site.go diff --git a/cmd/fetch_requirements.go b/cmd/fetch_requirements.go new file mode 100644 index 0000000..a898bd0 --- /dev/null +++ b/cmd/fetch_requirements.go @@ -0,0 +1,54 @@ +// Copyright © 2019 Red Hat +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + "gerrit.akraino.org/kni/installer/pkg/site" + "github.com/spf13/cobra" +) + +// fetchRequirementsCmd represents the fetch_requirements command +var fetchRequirementsCmd = &cobra.Command{ + Use: "fetch_requirements", + Short: "Command to fetch the requirements needed for a site", + Long: ``, + TraverseChildren: true, + Run: func(cmd *cobra.Command, args []string) { + // retrieve config values and start fetching + siteRepo, _ := cmd.Flags().GetString("site") + buildPath, _ := cmd.Flags().GetString("build_path") + if len(buildPath) == 0 { + // will generate a temporary directory + buildPath = fmt.Sprintf("%s/.kni", os.Getenv("HOME")) + } + + // define a site object and proceed with requirements fetch + s := site.New(siteRepo, buildPath) + s.DownloadSite() + s.FetchRequirements() + }, +} + +func init() { + rootCmd.AddCommand(fetchRequirementsCmd) + + fetchRequirementsCmd.Flags().StringP("site", "", "", "Url/path for site repository. Can be in any go-getter compatible format") + fetchRequirementsCmd.MarkFlagRequired("site") + fetchRequirementsCmd.Flags().StringP("build_path", "", "", "Directory to use as build path. If that not exists, the installer will generate a default directory") + +} diff --git a/pkg/requirements/requirements.go b/pkg/requirements/requirements.go new file mode 100644 index 0000000..1fe40e8 --- /dev/null +++ b/pkg/requirements/requirements.go @@ -0,0 +1,122 @@ +package requirements + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + getter "github.com/hashicorp/go-getter" +) + +// Requirement : Structure that contains the settings needed for managing a requirement +type Requirement struct { + binaryName string + sourceRepo string + buildPath string +} + +// New constructor for the generator +func New(binaryName string, sourceRepo string, buildPath string) Requirement { + r := Requirement{binaryName, sourceRepo, buildPath} + return r +} + +// download requirement from a tarball or folder +func (r Requirement) FetchRequirementFolder() { + // extract the tarball if exists + log.Println(fmt.Sprintf("Pulling %s tarball from %s", r.binaryName, r.sourceRepo)) + + extractDir := fmt.Sprintf("%s/%s_content", r.buildPath, r.binaryName) + client := &getter.Client{Src: r.sourceRepo, Dst: extractDir, Mode: getter.ClientModeAny} + err := client.Get() + if err != nil { + log.Fatal(fmt.Sprintf("Error cloning tarball repository: %s", err)) + os.Exit(1) + } + + // find the binary inside the extracted content + alternativeBinaryName := path.Base(r.sourceRepo) + err = filepath.Walk(extractDir, func(path string, info os.FileInfo, err error) error { + if (info.Name() == r.binaryName || info.Name() == alternativeBinaryName) && !info.IsDir() { + // we found the binary, move it. Give exec perms as well + finalName := fmt.Sprintf("%s/%s", r.buildPath, r.binaryName) + os.Rename(path, finalName) + os.Chmod(finalName, 0755) + os.RemoveAll(extractDir) + return nil + } + return nil + }) +} + +// generates the openshift binary +func (r Requirement) BuildOpenshiftBinary() { + extractDir := fmt.Sprintf("%s/src/github.com/openshift/installer", r.buildPath) + client := &getter.Client{Src: r.sourceRepo, Dst: extractDir, Mode: getter.ClientModeAny} + err := client.Get() + if err != nil { + log.Fatal(fmt.Sprintf("Error cloning tarball repository: %s", err)) + os.Exit(1) + } + + // build the openshift binary + cmd := exec.Command("hack/build.sh") + cmd.Dir = extractDir + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "TAGS=libvirt") + cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", r.buildPath)) + + var stdBuffer bytes.Buffer + mw := io.MultiWriter(os.Stdout, &stdBuffer) + cmd.Stdout = mw + cmd.Stderr = mw + + err = cmd.Run() + if err != nil { + log.Fatal(fmt.Sprintf("Error building binary: %s - %s", err, stdBuffer.String())) + os.Exit(1) + } + log.Println(stdBuffer.String()) + + // copy the generated binary to the build directory + cmd = exec.Command("cp", fmt.Sprintf("%s/bin/openshift-install", extractDir), r.buildPath) + err = cmd.Run() + if err != nil { + log.Fatal(fmt.Sprintf("Error copying installer to buid path: %s", err)) + os.Exit(1) + } + log.Println(fmt.Sprintf("Installer is available on %s/openshift-install", r.buildPath)) +} + +// download a requirement from a git repo and build it +func (r Requirement) FetchRequirementGit() { + if r.binaryName == "openshift-install" { + r.BuildOpenshiftBinary() + } else { + log.Fatal(fmt.Sprintf("Build of binary %s is not supported", r.binaryName)) + os.Exit(1) + } +} + +// downloads an individual requirement +func (r Requirement) FetchRequirement() { + log.Println(fmt.Sprintf("Downloading %s requirement from %s", r.binaryName, r.sourceRepo)) + + // first check if the binary already exists + binaryPath := fmt.Sprintf("%s/%s", r.buildPath, r.binaryName) + if _, err := os.Stat(binaryPath); err == nil { + log.Println(fmt.Sprintf("Using existing %s", binaryPath)) + } else if os.IsNotExist(err) { + if strings.Contains(r.sourceRepo, ".git") { + r.FetchRequirementGit() + } else { + r.FetchRequirementFolder() + } + } +} diff --git a/pkg/site/site.go b/pkg/site/site.go new file mode 100644 index 0000000..0da768a --- /dev/null +++ b/pkg/site/site.go @@ -0,0 +1,116 @@ +package site + +import ( + "bufio" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "strings" + + "gerrit.akraino.org/kni/installer/pkg/requirements" + getter "github.com/hashicorp/go-getter" + "gopkg.in/yaml.v2" +) + +// Site : Structure that contains the settings needed for managing a site +type Site struct { + siteRepo string + siteName string + buildPath string +} + +// New constructor for the generator +func New(siteRepo string, buildPath string) Site { + // given a site repo, extract the site name from the path + baseName := path.Base(siteRepo) + suffixList := [2]string{".git", ".tar.gz"} + + siteName := baseName + for _, suffix := range suffixList { + siteName = strings.TrimSuffix(siteName, suffix) + } + + s := Site{siteRepo, siteName, buildPath} + return s +} + +// given a site repo, downloads the content and places into buildPath +func (s Site) DownloadSite() { + // Clone the site repository + log.Println(fmt.Sprintf("Cloning the site repository from %s", s.siteRepo)) + siteBuildPath := fmt.Sprintf("%s/%s/site", s.buildPath, s.siteName) + client := &getter.Client{Src: s.siteRepo, Dst: siteBuildPath, Mode: getter.ClientModeAny} + err := client.Get() + if err != nil { + log.Fatal(fmt.Sprintf("Error cloning site repository: %s", err)) + } + +} + +// using the downloaded site content, fetches (and builds) the specified requirements +func (s Site) FetchRequirements() { + log.Println(fmt.Sprintf("Downloading requirements for %s", s.siteName)) + siteBuildPath := fmt.Sprintf("%s/%s", s.buildPath, s.siteName) + + // searches for file containing the profile of the blueprint + profileFile := fmt.Sprintf("%s/site/00_install-config/kustomization.yaml", siteBuildPath) + + if _, err := os.Stat(profileFile); err == nil { + // parse yaml and extract base + yamlContent, err := ioutil.ReadFile(profileFile) + if err != nil { + log.Fatal(fmt.Sprintf("Error reading profile file: %s", err)) + os.Exit(1) + } + + profileSettings := &map[string][]interface{}{} + err = yaml.Unmarshal(yamlContent, &profileSettings) + if err != nil { + log.Fatal(fmt.Sprintf("Error parsing profile yaml file: %s", err)) + os.Exit(1) + } + bases := (*profileSettings)["bases"] + profileRepo := fmt.Sprintf("%s", bases[0]) + + // given the profile repo, we need to get the full path without file, and clone it + profileBits := strings.Split(profileRepo, "/") + profileName := profileBits[len(profileBits)-2] + profilePath := strings.TrimSuffix(profileRepo, profileBits[len(profileBits)-1]) + + profileBuildPath := fmt.Sprintf("%s/%s", siteBuildPath, profileName) + log.Println(fmt.Sprintf("Downloading profile repo from %s into %s", profilePath, profileBuildPath)) + client := &getter.Client{Src: profilePath, Dst: profileBuildPath, Mode: getter.ClientModeAny} + err = client.Get() + if err != nil { + log.Fatal(fmt.Sprintf("Error cloning profile repository: %s", err)) + } + + // read yaml from requirements and start fetching the bits + requirementsFile := fmt.Sprintf("%s/requirements.yaml", profileBuildPath) + file, err := os.Open(requirementsFile) + if err != nil { + log.Fatal("Error reading requirements file") + os.Exit(1) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + requirementsLine := scanner.Text() + + // requirements is composed of binary and source + requirementsBits := strings.SplitN(strings.TrimSpace(requirementsLine), ":", 2) + r := requirements.New(strings.TrimSpace(requirementsBits[0]), strings.TrimSpace(requirementsBits[1]), fmt.Sprintf("%s/requirements", siteBuildPath)) + r.FetchRequirement() + } + + // remove profile folder + os.RemoveAll(profileBuildPath) + + } else if os.IsNotExist(err) { + log.Fatal(fmt.Sprintf("File %s does not exist, exiting", profileFile)) + os.Exit(1) + } +} -- 2.16.6