tools/yum.py

   1 # Copyright 2019 Nokia
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import logging
  18
  19 from tools.executor import run
  20 from tools.rpm import RpmData
  21
  22
  23 class YumRpm(RpmData):
  24     @property
  25     def arch(self):
  26         return self['Arch']
  27
  28
  29 class YumInfoParserException(BaseException):
  30     pass
  31
  32
  33 class YumDownloaderException(BaseException):
  34     def __init__(self, msg, failed_packages):
  35         super(YumDownloaderException, self).__init__()
  36         self.msg = msg
  37         self.failed_packages = failed_packages
  38
  39     def __str__(self):
  40         return self.msg
  41
  42
  43 class YumInfoParser(object):
  44     """
  45     Parse 'yum info' output
  46     """
  47
  48     def parse_file(self, yum_info_installed_file_path):
  49         with open(yum_info_installed_file_path, 'r') as f:
  50             return self.parse_installed(f.read())
  51
  52     def parse_installed(self, yum_info_installed):
  53         return self._parse_rpms_with_regexp(yum_info_installed, r'\nInstalled Packages\n')
  54
  55     def parse_available(self, yum_info_installed):
  56         return self._parse_rpms_with_regexp(yum_info_installed, r'Available Packages\n')
  57
  58     def _parse_rpms_with_regexp(self, yum_output, regexp):
  59         parsed_output = self._split_yum_output_with(yum_output, regexp)
  60         return [self.parse_package(pkg) for pkg in parsed_output[1].split('\n\n') if pkg]
  61
  62     @staticmethod
  63     def _split_yum_output_with(output, regexp):
  64         parsed_output = re.split(regexp, output)
  65         if len(parsed_output) != 2:
  66             raise YumInfoParserException(
  67                 '{} not found from output: {}'.format(repr(regexp), output[:1000]))
  68         return parsed_output
  69
  70     @staticmethod
  71     def parse_package(yum_info_output):
  72         result = YumRpm()
  73         current_key = None
  74         for line in re.findall(r'^(.+?) : (.*)$', yum_info_output, re.MULTILINE):
  75             parsed_key = line[0].strip()
  76             parsed_value = line[1].rstrip(' ')
  77             if parsed_key:
  78                 result[parsed_key] = parsed_value
  79                 current_key = parsed_key
  80             elif current_key in ['License', 'Summary']:
  81                 result[current_key] = result[current_key] + ' ' + parsed_value
  82             else:
  83                 result[current_key] = result[current_key] + '\n' + parsed_value
  84         return result
  85
  86
  87 class Yum(object):
  88     def __init__(self):
  89         self.config = YumConfig(filename='tmp_yum.conf')
  90
  91     @classmethod
  92     def clean_and_remove_cache(cls):
  93         cls.clean_all()
  94         cls.remove_cache_dir()
  95
  96     @classmethod
  97     def clean_all(cls):
  98         run(['yum', 'clean', 'all'], raise_on_stderr=False)
  99
 100     @classmethod
 101     def remove_cache_dir(cls):
 102         run(['rm', '-rf', '/var/cache/yum'], raise_on_stderr=True)
 103
 104     @classmethod
 105     def read_available_pkgs(cls, name, url):
 106         filename = 'tmp_yum.conf'
 107         yum_tmp_conf = YumConfig()
 108         yum_tmp_conf.add_repository(name, url)
 109         with open(filename, 'w') as f:
 110             f.write(yum_tmp_conf.render())
 111         cmd = ['yum',
 112                '--config={}'.format(filename),
 113                '--showduplicates',
 114                '--setopt=keepcache=0',
 115                '--disablerepo=*',
 116                '--enablerepo={}'.format(name),
 117                'info',
 118                'available']
 119         return run(cmd, raise_on_stderr=False).stdout
 120
 121     def add_repo(self, name, url):
 122         self.config.add_repository(name, url)
 123
 124     def read_all_packages(self):
 125         self.config.write()
 126         logging.debug('Yum config:\n{}'.format(self.config))
 127         cmd = ['yum',
 128                '--config={}'.format(self.config.filename),
 129                '--showduplicates',
 130                '--setopt=keepcache=0',
 131                '--enablerepo=*',
 132                'info',
 133                'available']
 134         return run(cmd, raise_on_stderr=False).stdout
 135
 136
 137 class YumDownloader(object):
 138     def download(self, rpms, repositories, to_dir=None, source=False):
 139         logging.debug('Downloading {} RPMs from repositories: {}'.format(len(rpms),
 140                                                                          [r['name'] for r in
 141                                                                           repositories]))
 142         result = self._download(rpms, repositories, to_dir=to_dir, source=source)
 143         downloaded_rpms = [rpm.rstrip('.rpm') for rpm in os.listdir(to_dir)]
 144         not_downloaded_rpms = [rpm for rpm in rpms if rpm not in downloaded_rpms]
 145         if len(rpms) != len(downloaded_rpms):
 146             logging.debug('Downloaded {}/{} RPMs: {}'.format(len(downloaded_rpms), len(rpms),
 147                                                              downloaded_rpms))
 148             # Not precise way to list not downloaded RPMs - RPM name may not match the metadata
 149             # We should read "rpm -qip" for each rpm and parse it
 150             raise YumDownloaderException(
 151                 'Failed to download {}/{} RPMs: {}.\nYumdownloader result: {}'.format(
 152                     len(not_downloaded_rpms), len(rpms), not_downloaded_rpms, str(result)),
 153                 not_downloaded_rpms
 154             )
 155
 156     @staticmethod
 157     def _download(rpms, repositories, to_dir=None, source=False):
 158         filename = 'tmp_yum.conf'
 159         yum_tmp_conf = YumConfig()
 160         for r in repositories:
 161             yum_tmp_conf.add_repository(r['name'], r['baseurl'])
 162         with open(filename, 'w') as f:
 163             f.write(yum_tmp_conf.render())
 164         logging.debug('Downloading RPMs: {}'.format(rpms))
 165         cmd = ['yumdownloader']
 166         if to_dir is not None:
 167             cmd += ['--destdir={}'.format(to_dir)]
 168         cmd += ['--config={}'.format(filename),
 169                 '--disablerepo=*',
 170                 '--enablerepo={}'.format(','.join([r['name'] for r in repositories])),
 171                 '--showduplicates']
 172         if source:
 173             cmd.append('--source')
 174         cmd += rpms
 175         return run(cmd, raise_on_stderr=False)
 176
 177
 178 class YumConfig(object):
 179     def __init__(self, filename=None):
 180         self.filename = filename
 181         self.config = ['[main]',
 182                        '#cachedir=/var/cache/yum/$basearch/$releasever',
 183                        'reposdir=/foo/bar/xyz',
 184                        'keepcache=0',
 185                        'debuglevel=2',
 186                        '#logfile=/var/log/yum.log',
 187                        'exactarch=1',
 188                        'obsoletes=1',
 189                        'gpgcheck=0',
 190                        'plugins=1',
 191                        'installonly_limit=5',
 192                        'override_install_langs=en_US.UTF-8',
 193                        'tsflags=nodocs']
 194         self.repositories = []
 195
 196     def add_repository(self, name, url, exclude=None):
 197         repo = ['[{}]'.format(name),
 198                 'name = ' + name,
 199                 'baseurl = ' + url,
 200                 'enabled = 1',
 201                 'gpgcheck = 0']
 202         if exclude is not None:
 203             repo.append('exclude = ' + exclude)
 204         self.repositories.append(repo)
 205
 206     def __str__(self):
 207         return self.render()
 208
 209     def render(self):
 210         blocks = ['\n'.join(self.config)] + ['\n'.join(repo) for repo in self.repositories]
 211         return '\n\n'.join(blocks)
 212
 213     def write(self):
 214         with open(self.filename, 'w') as f:
 215             f.write(self.render())