kgenerate
first published on Dec 14, 2017

This python script is used to create our final Kubernetes cluster as discussed here

#!/usr/bin/env python
from __future__ import print_function
import argparse
import sys
import os
import string
import operator
import errno
import uuid
import re
from collections import namedtuple


class Tmpl():
    "Utility class to generate, write and transpile a template"

    def __init__(self, tmpl, dst="", master=False, other=False, nany=False,
                 etcd=False, toplevel=False, subtemplate=False, ct=False):
        with open("./templates/" + tmpl + ".template") as f:
            self.template = f.read()
        self.destination = dst
        self.toplevel = toplevel
        self.ct = ct
        if ct:
            self.toplevel = True

        if nany:
            self.master = True
            self.other = True
            self.etcd = True
        else:
            self.master = master
            self.other = other
            self.etcd = etcd

    def generate(self, subs):
        self.content = string.Template(self.template).substitute(subs)
        return self.content

    def tokengenerate(self, node):
        token = str(uuid.uuid4()).replace('-', '')
        self.content = string.Template(self.template).substitute({
            "master": node,
            "token": token,
        })
        return token

    def write(self, node="", master=False, other=False, nany=False,
              etcd=False, ct=False):
        self.node = node
        if (nany or
            (master and self.master) or
            (other and self.other) or
                (etcd and self.etcd)):
            dst = "./out"
            if not self.toplevel:
                if node == "":
                    raise ValueError("Unset node for a non-toplevel template")
                dst = dst + "/" + node

            if self.ct:
                dst = dst + "/" + node + ".ct.tmpl"
            else:
                if self.destination == "":
                    raise ValueError("No destination set!")
                dst = dst + "/" + self.destination
            with open(dst, "w") as t:
                t.write(self.content)
        else:
            raise ValueError("Mismatch with the template declaration and "
                             "usage, did not write %s" % self.content)

    def transpile(self):
        if not self.ct:
            raise ValueError("Cannot transpile a non-ct template")

        if self.node == "":
            raise ValueError("Unset node")

        with open("./out/" + self.node + ".ct", "w") as f:
            for x in self.content.split('\n'):
                try:
                    col = x.index("###")
                    colspace = " " * (col - 1)
                except Exception:
                    f.write(x + "\n")
                    continue

                subfile = x[col + 3:]
                with open(subfile) as c:
                    subcontents = c.read().split('\n')

                if x[col - 1] == '|':
                    for cline in subcontents:
                        f.write(colspace + cline + "\n")
                elif x[col - 1] == '-':
                    f.write(x[:(col - 1)] + "".join(subcontents) + "\n")
                else:
                    raise ValueError("Unknown qualifier in %s", x)


def invocation():
    "Takes the command line parameters and validates them"

    parser = argparse.ArgumentParser(
        description='''Create the cluster configuration files.
                       Will create a .ct.tmpl for each .cfg file.
                       Maximum 10 nodes/images total.''')

    parser.add_argument(
        '-e', '--etcd', metavar='E', type=int, required=True,
        help='number of etcd nodes')
    parser.add_argument(
        '-c', '--cluster', metavar='C', nargs="+",
        required=True, help='''space separated image names, without .cfg,
                                will be in order master, etcd nodes, other
                                nodes. All image files in the directory need
                                to be listed here''')
    parser.add_argument(
        '-k', '--kube', metavar='K', type=str, required=True,
        help='''hyperkube tag to use, see https://quay.io/repository/
        coreos/hyperkube?tag=latest&tab=tags for available tags''')
    args = parser.parse_args()

    tag = args.kube
    if not re.match('^v[0-9]+\.[0-9]+\.[0-9]$', tag):
        raise AttributeError("The passed tag does not look correct,"
                             " expecting it in vx.y.z format"
                             "(%s)\n" % tag)

    xendir = os.getenv("XENDIR", "")
    if xendir == "":
        raise AttributeError("XENDIR must be set")
    pwd = os.getcwd()
    guest = os.path.basename(pwd)
    expected = ''.join([xendir, "/guests/", guest])
    if pwd != expected:
        raise AttributeError("You should be in a guests directory!"
                             " (am in %s, expected %s)" % (pwd, expected))

    if args.etcd < 1:
        raise AttributeError("At least one etcd node is required (%d)" %
                             args.etcd)

    with open("/var/lib/dnsmasq/virbr1/hostsfile") as f:
        t = f.read().split('\n')
        dnsmasq = [s for s in t if len(s) > 0 and '#' not in s]
        ips = dict(x.split(',') for x in dnsmasq)

    Cfg = namedtuple("Cfg", "tag cluster netcd nodes ips master etcd other")
    return Cfg(
        tag,
        guest,
        args.etcd + 1,
        args.cluster,
        ips,
        args.cluster[0],
        args.cluster[1:args.etcd + 1],
        args.cluster[args.etcd + 1:],
    )


def assemble(args):
    '''Create the nodes list from the local .cfg files, as well as the
       initial common template substitutions'''

    nodeips = {}
    for entry in os.listdir('.'):
        if not entry.endswith('.cfg') or not os.path.isfile(entry):
            continue
        c = entry[:-4]
        if c not in args.nodes:
            raise ValueError("Image %s is not in the cluster list!\n" % c)
        with open(entry) as f:
            t = f.read().split('\n')
            for x in t:
                if x.startswith('name'):
                    name = x.split('=')[1].strip().replace('"', '')
                elif x.startswith('vif'):
                    cut = x.index('mac')
                    t = x[cut:]
                    cut = t.index(',')
                    t = t[:cut]
                    mac = t.split('=')[1].strip().replace('"', '')

            if (not name.startswith(args.cluster + "-") or
                    name[len(args.cluster) + 1:] not in args.nodes):
                raise ValueError("Configuration file %s has hostname %s which"
                                 "is not in the cluster" % (entry, name))
            nodeips[c] = mac

    if len(nodeips) < args.netcd or args.netcd > 9:
        raise ValueError("%d images with %d etcd nodes is not going to"
                         " work\n" % (len(nodeips), args.netcd - 1))

    for k, v in nodeips.items():
        if v not in args.ips:
            raise ValueError("Expected to find %s in %s\n" % (v, args.ips))

        nodeips[k] = args.ips[v]

    subs = {
        # Will create
        # solar-mercury=https://192.168.100.21:2380,solar-venus=https://192...
        "etcdlist": ','.join([args.cluster + '-' + x + '=https://' +
                              nodeips[x] + ':2380'
                              for x in args.etcd]),
        # will create https://192.168.100.21:2379,https://192.168.100.2...
        "etcdiplist": ','.join(['https://' + nodeips[x] + ':2379'
                                for x in args.etcd]),
        "guest": args.cluster,
        "home": os.getenv("HOME"),
    }
    return (nodeips, subs)


def main():
    try:
        # Get the sanitized args
        args = invocation()
        (nodeips, subs) = assemble(args)

        # Set up the needed templates
        library = {
            "apiserver": Tmpl(
                "apiserver", dst="apiserver.yaml", master=True),
            "controller": Tmpl(
                "controller", dst="controller.yaml", master=True),
            "etcd": Tmpl("etcd"),
            "flannel": Tmpl("flannel-setup"),
            "flanneld": Tmpl("flanneld"),
            "kube-controller-manager": Tmpl("kube-controller-manager",
                                            "kube-controller-manager.cfg",
                                            master=True),
            "kube-proxy": Tmpl("kube-proxy", "kube-proxy.cfg", nany=True),
            "kube-scheduler": Tmpl("kube-scheduler",
                                   "kube-scheduler.cfg", master=True),
            "kubelet": Tmpl("kubelet", dst="kubelet.cfg", nany=True),
            "node-etcd": Tmpl("node-etcd", etcd=True, ct=True),
            "node-master": Tmpl("node-master", master=True, ct=True),
            "node-other": Tmpl("node-other", other=True, ct=True),
            "profile": Tmpl("profile", dst="profile.txt",
                            toplevel=True),
            "proxy": Tmpl(
                "proxy", dst="proxy.yaml", nany=True, toplevel=True),
            "scheduler": Tmpl(
                "scheduler", dst="scheduler.yaml", master=True),
            "storage": Tmpl("storage"),
            "storage-etcd": Tmpl("storage-etcd"),
            "storage-master": Tmpl("storage-master"),
            "systemd": Tmpl("systemd"),
        }

        # Prepare the output directory
        try:
            os.makedirs("./out")
            for x in args.nodes:
                os.makedirs("./out/" + x)
        except OSError as e:
            if (e.errno == errno.EEXIST):
                pass

        # First a non-template file
        with open("./out/hosts.txt", "w") as t:
            t.write('''# Autogenerated
127.0.0.1    localhost
::1          localhost
''')
            for x in sorted(nodeips.items(), key=operator.itemgetter(1)):
                t.write(x[1] + "  " + args.cluster + "-" + x[0] +
                        ".cluster.local" + "  " + args.cluster + "-" +
                        x[0] + "\n")

        # Prepare the global substitutions, later substitutions might depend
        # on the former.
        subs["master"] = nodeips[args.master]
        subs["hypertag"] = args.tag

        # We have two flanneld sections depending if it's an etcd node or not
        subs["flanneldsetup"] = library["flannel"].generate(subs)
        flanneld_with_etcd_setup = library["flanneld"].generate(subs)
        subs["flanneldsetup"] = ""
        subs["flanneld"] = library["flanneld"].generate(subs)

        # Time to start writing files
        library["profile"].generate(subs)
        library["profile"].write(nany=True)

        # First create the basic_auth csv file with a random password
        password = str(uuid.uuid4()).replace('-', '')
        with open("./out/basic_auth.csv", "w") as t:
            t.write(password)
            t.write(",admin,admin,system:masters\n")

        # Now all the other tokens, first the admin token
        token = str(uuid.uuid4()).replace('-', '')
        tokens = [','.join([token,
                            "admin",
                            "admin",
                            "system:masters"])]

        # then kubelet and kube-proxy, for all nodes
        for x in args.nodes:
            tokens.append(','.join([library["kubelet"].tokengenerate(
                nodeips[args.master]),
                "system:node:" + args.cluster + "-" + x,
                "system:node:" + args.cluster + "-" + x,
                "system:nodes"]))
            library["kubelet"].write(node=x, nany=True)

            tokens.append(','.join([library["kube-proxy"].tokengenerate(
                nodeips[args.master]),
                "system:kube-proxy",
                "system:kube-proxy",
                "system:node-proxier"]))
            library["kube-proxy"].write(node=x, nany=True)

        # Finally scheduler and controller manager, which are master-only
        tokens.append(','.join([library["kube-scheduler"].tokengenerate(
            nodeips[args.master]),
            "system:kube-scheduler",
            "system:kube-scheduler",
            "system:kube-scheduler"]))
        library["kube-scheduler"].write(node=args.master, master=True)

        tokens.append(','.join([
            library["kube-controller-manager"].tokengenerate(
                nodeips[args.master]),
            "system:kube-controller-manager",
            "system:kube-controller-manager",
            "system:kube-controller-manager"]))
        library["kube-controller-manager"].write(node=args.master,
                                                 master=True)

        # All the tokens we have created have to go in their own file
        with open('./out/known_tokens.csv', "w") as t:
            for x in tokens:
                t.write(x)
                t.write("\n")

        # Master-only templates
        for t in ["apiserver", "proxy", "controller", "scheduler"]:
            library[t].generate(subs)
            library[t].write(node=args.master, master=True)

        # Master:
        subs["hostname"] = args.master
        subs["hostip"] = nodeips[args.master]
        subs["storage"] = library["storage"].generate(subs)
        subs["storagemaster"] = library["storage-master"].generate(subs)
        subs["systemd"] = library["systemd"].generate(subs)
        library["node-master"].generate(subs)
        library["node-master"].write(node=args.master, master=True)
        library["node-master"].transpile()

        # Other nodes
        for x in args.other:
            subs["hostname"] = x
            subs["hostip"] = nodeips[x]
            subs["storage"] = library["storage"].generate(subs)
            library["node-other"].generate(subs)
            library["node-other"].write(node=x, other=True)
            library["node-other"].transpile()

        # Etcd nodes
        subs["flanneld"] = flanneld_with_etcd_setup
        subs["systemd"] = library["systemd"].generate(subs)
        for x in args.etcd:
            subs["hostname"] = x
            subs["hostip"] = nodeips[x]
            subs["storage"] = library["storage"].generate(subs)
            subs["storage"] = library["storage-etcd"].generate(subs)
            subs["etcd"] = library["etcd"].generate(subs)
            library["node-etcd"].generate(subs)
            library["node-etcd"].write(node=x, etcd=True)
            library["node-etcd"].transpile()

        # And we are done!
        print("Created templates for a cluster with:")
        print("Master:   %-20s (%s)" % (args.cluster + "-" + args.master,
                                        nodeips[args.master]))
        for x in args.etcd:
            print("Etcd:     %-20s (%s)" % (args.cluster + "-" + x,
                                            nodeips[x]))
        for x in args.other:
            print("Node:     %-20s (%s)" % (args.cluster + "-" + x,
                                            nodeips[x]))

    except Exception as e:
        print("Error: %s" % e)
        sys.exit(1)


if __name__ == '__main__':
    main()
Changelog:
  • Initial release - 2017-12-14