site/site30/baremetal/calico-ip-rules.yaml

   1 ---
   2 ##############################################################################
   3 # Copyright © 2018 AT&T Intellectual Property. All rights reserved.          #
   4 #                                                                            #
   5 # Licensed under the Apache License, Version 2.0 (the "License"); you may    #
   6 # not use this file except in compliance with the License.                   #
   7 #                                                                            #
   8 # You may obtain a copy of the License at                                    #
   9 #       http://www.apache.org/licenses/LICENSE-2.0                           #
  10 #                                                                            #
  11 # Unless required by applicable law or agreed to in writing, software        #
  12 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT  #
  13 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.           #
  14 # See the License for the specific language governing permissions and        #
  15 # limitations under the License.                                             #
  16 ##############################################################################
  17
  18 schema: 'drydock/BootAction/v1'
  19 metadata:
  20   schema: 'metadata/Document/v1'
  21   name: calico-ip-rules
  22   storagePolicy: 'cleartext'
  23   layeringDefinition:
  24     abstract: false
  25     layer: site
  26   labels:
  27     application: 'drydock'
  28   substitutions:
  29     - src:
  30         schema: pegleg/CommonAddresses/v1
  31         name: common-addresses
  32         path: .kubernetes.pod_cidr
  33       dest:
  34         path: .assets[0].data
  35         pattern: DH_SUB_POD_CIDR
  36 data:
  37   signaling: false
  38   assets:
  39     - path: /etc/systemd/system/configure-ip-rules.service
  40       type: unit
  41       permissions: '444'
  42       data: |-
  43         [Unit]
  44         Description=IP Rules Initialization Service
  45         After=network-online.target local-fs.target
  46
  47         [Service]
  48         Type=simple
  49         #ExecStart=/opt/configure-ip-rules.sh -i bond1.2406 -c DH_SUB_POD_CIDR -o 10.34.0.0/15 -s 135.21.157.32/29
  50         #ExecStart=/opt/configure-ip-rules.sh -i bond0.44 -c DH_SUB_POD_CIDR -o 10.99.0.0/16 -s 172.29.1.0/24
  51         ExecStart=/opt/configure-ip-rules.sh -g 172.29.1.1 -c 10.99.0.0/16 -s 172.29.1.136/29
  52
  53
  54
  55         [Install]
  56         WantedBy=multi-user.target
  57       data_pipeline:
  58         - utf8_decode
  59     - path: /opt/configure-ip-rules.sh
  60       type: file
  61       permissions: '700'
  62       data_pipeline:
  63         - utf8_decode
  64       data: |-
  65         #!/bin/bash
  66         set -ex
  67
  68         function usage() {
  69             cat <<EOU
  70         Options are:
  71
  72           -c POD_CIDR     The pod CIDR for the Kubernetes cluster, e.g. 10.97.0.0/16
  73           -i INTERFACE    The interface for internal pod traffic, e.g. bond1.2006
  74           -o OVERLAP_CIDR (optional) This CIDR will be routed via the VRRP IP on
  75                           INTERFACE.  It is used to provide a work around when
  76                           complete Calico routes cannot be received via BGP.
  77                           e.g. 10.96.0.0/15.  NOTE: This must include the POD_CIDR.
  78           -s SERVICE_CIDR (optional) A routable CIDR to configure for ingress, maas,
  79                           e.g. 135.21.99.192/29
  80         EOU
  81         }
  82
  83         SERVICE_CIDR=
  84         OVERLAP_CIDR=
  85
  86         while getopts ":c:hi:o:s:" o; do
  87             case "${o}" in
  88                 c)
  89                     POD_CIDR=${OPTARG}
  90                     ;;
  91                 h)
  92                     usage
  93                     exit 0
  94                     ;;
  95                 i)
  96                     INTERFACE=${OPTARG}
  97                     ;;
  98                 o)
  99                     OVERLAP_CIDR=${OPTARG}
 100                     ;;
 101                 s)
 102                     SERVICE_CIDR=${OPTARG}
 103                     ;;
 104                 \?)
 105                     echo "Unknown option: -${OPTARG}" >&2
 106                     exit 1
 107                     ;;
 108                 :)
 109                     echo "Missing argument for option: -${OPTARG}" >&2
 110                     exit 1
 111                     ;;
 112                 *)
 113                     echo "Unimplemented option: -${OPTARG}" >&2
 114                     exit 1
 115                     ;;
 116             esac
 117         done
 118         shift $((OPTIND-1))
 119
 120         if [ "x$POD_CIDR" == "x" ]; then
 121             echo "Missing pod CIDR, e.g -c 10.97.0.0/16" >&2
 122             usage
 123             exit 1
 124         fi
 125
 126         if [ "x$INTERFACE" == "x" ]; then
 127             echo "Missing interface, e.g. -i bond1.2006" >&2
 128             usage
 129             exit 1
 130         fi
 131
 132         while ! ip route list dev "${INTERFACE}" > /dev/null; do
 133             echo Waiting for device "${INTERFACE}" to be ready. >&2
 134             sleep 5
 135         done
 136
 137         intra_vrrp_ip=$(ip route list dev "${INTERFACE}" | awk '($2~/via/){print $3}' | head -n 1)
 138
 139         TABLE="1500"
 140
 141         # Setup a routing table for traffic from service IPs
 142         ip route flush table "${TABLE}"
 143         ip route add default via "${intra_vrrp_ip}" table "${TABLE}"
 144
 145         if [ "x$OVERLAP_CIDR" != "x" ]; then
 146             # NOTE(mb874d): This is a work-around for nodes not receiving complete
 147             # routes via BGP.  It may also be required for brownfield large sites.
 148             ip route add "${OVERLAP_CIDR}" via "${intra_vrrp_ip}"
 149         fi
 150
 151         if [ "x$SERVICE_CIDR" != "x" ]; then
 152             # Traffic from the service IPs to pods should use the pod network.
 153             ip rule add \
 154                 from "${SERVICE_CIDR}" \
 155                 to "${POD_CIDR}" \
 156                 lookup main \
 157                 pref 10000
 158             # Other traffic from service IPs should only use the VRRP IP
 159             ip rule add \
 160                 from "${SERVICE_CIDR}" \
 161                 lookup "${TABLE}" \
 162                 pref 10100
 163         fi
 164 ...