From c9bbbd297a23b8f2a02a69e3e6eec084372600bb Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 1 Feb 2023 00:30:03 +0100 Subject: [PATCH] Port Puppet compiler to Python --- .../roles/apply_puppet/tasks/main.yml | 118 ++++++-------- personal_infra/up.py | 152 ++++++++++++++++++ 2 files changed, 202 insertions(+), 68 deletions(-) create mode 100755 personal_infra/up.py diff --git a/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml b/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml index 0ee291e..3512717 100644 --- a/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml +++ b/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml @@ -1,97 +1,77 @@ --- -- name: create local temporary directory - tempfile: - state: directory - path: "{{ inventory_dir }}/tmp" - register: local_temp - delegate_to: 127.0.0.1 +- name: clean puppet build directory + local_action: + module: file + path: "{{ inventory_dir }}/build/puppet" + state: absent + run_once: True tags: puppet_fast -- name: create data directory in local temp - file: - path: "{{ local_temp.path }}/data" +- name: create puppet build directories + local_action: + module: file + path: "{{ inventory_dir }}/{{ item }}" + state: directory + loop: + - build/puppet/global_vars + - build/puppet/host_vars + - build/puppet/facts + run_once: True + tags: puppet_fast +- name: create puppet build host vars directories + local_action: + module: file + path: "{{ inventory_dir }}/build/puppet/host_vars/{{ inventory_hostname }}" state: directory - delegate_to: 127.0.0.1 - tags: puppet_fast -- name: create hiera.yaml - copy: - dest: "{{ local_temp.path }}/hiera.yaml" - content: | - version: 5 - hierarchy: - - name: hostvars - path: hostvars.json - data_hash: json_data - - name: this - path: this.json - data_hash: json_data - delegate_to: 127.0.0.1 tags: puppet_fast - name: dump hostvars - copy: - dest: "{{ local_temp.path }}/data/hostvars.json" + local_action: + module: copy + dest: "{{ inventory_dir }}/build/puppet/global_vars/hostvars.json" content: "{'hostvars': {{ hostvars }} }" - delegate_to: 127.0.0.1 + run_once: True tags: puppet_fast - name: dump this - copy: - dest: "{{ local_temp.path }}/data/this.json" + local_action: + module: copy + dest: "{{ inventory_dir }}/build/puppet/host_vars/{{ inventory_hostname }}/this.json" content: "{{ hostvars[inventory_hostname] }}" - delegate_to: 127.0.0.1 tags: puppet_fast -- name: install epel - package: - name: epel-release - when: ansible_distribution_file_variety == 'RedHat' -- name: install packages - package: - name: - - puppet - - unzip - name: get facts command: facter -y register: facter_output tags: puppet_fast -- name: create facts directory in local temp - file: - path: "{{ local_temp.path }}/yaml/facts" - state: directory - delegate_to: 127.0.0.1 - tags: puppet_fast - name: dump facts - copy: - dest: "{{ local_temp.path }}/yaml/facts/{{ inventory_hostname }}.yaml" - content: "--- !ruby/object:Puppet::Node::Facts\nvalues:\n {{ facter_output.stdout | indent(width=2) }}" + local_action: + module: copy + dest: "{{ inventory_dir }}/build/puppet/facts/{{ inventory_hostname }}.yaml" + content: "{{ facter_output.stdout }}" delegate_to: 127.0.0.1 tags: puppet_fast -- name: compile catalogs - command: puppet catalog compile --modulepath={{ inventory_dir }}/puppet/modules --hiera_config={{ local_temp.path }}/hiera.yaml --manifest={{ inventory_dir }}/puppet/site --terminus compiler --vardir {{ local_temp.path }}/ --facts_terminus yaml {{ inventory_hostname }} +- name: compile puppet catalogs + local_action: + module: command + cmd: "{{ inventory_dir }}/up.py {{ inventory_dir }}/build/puppet {{ inventory_dir }}/puppet/modules {{ inventory_dir }}/puppet/site {% for host in ansible_play_batch %}{{ host }} {% endfor %}" + tags: puppet_fast + run_once: True +- name: package catalog + archive: + path: "{{ inventory_dir }}/build/puppet/build/output/{{ inventory_hostname }}" + dest: "{{ inventory_dir }}/build/puppet/puppet_catalog_{{ inventory_hostname }}.zip" + format: zip delegate_to: 127.0.0.1 - register: catalog tags: puppet_fast - name: create remote temporary directory tempfile: state: directory register: remote_temp tags: puppet_fast -- name: write catalog - copy: - dest: "{{ remote_temp.path }}/catalog.json" - content: "{{ catalog.stdout | regex_replace('\\A.*?\\n', multiline=True) }}" - tags: puppet_fast -- name: package modules - archive: - path: ../puppet/modules - dest: "{{ local_temp.path }}/puppet_modules.zip" - format: zip - delegate_to: 127.0.0.1 - tags: puppet_fast -- name: unpackage modules +- name: unpackage catalog unarchive: - src: "{{ local_temp.path }}/puppet_modules.zip" + src: "{{ inventory_dir }}/build/puppet/puppet_catalog_{{ inventory_hostname }}.zip" dest: "{{ remote_temp.path }}" tags: puppet_fast - name: preview catalog - command: puppet apply --catalog {{ remote_temp.path }}/catalog.json --noop --test --modulepath={{ remote_temp.path }}/modules/ + command: puppet apply --catalog {{ remote_temp.path }}/{{ inventory_hostname }}/catalog.json --noop --test --modulepath={{ remote_temp.path }}/{{ inventory_hostname }}/modules/ register: catalog_apply tags: puppet_fast - name: display catalog preview stdout @@ -106,7 +86,7 @@ pause: tags: pause - name: apply catalog - command: puppet apply --catalog {{ remote_temp.path }}/catalog.json --modulepath={{ remote_temp.path }}/modules/ + command: puppet apply --catalog {{ remote_temp.path }}/{{ inventory_hostname }}/catalog.json --modulepath={{ remote_temp.path }}/{{ inventory_hostname }}/modules/ register: catalog_apply tags: puppet_fast - name: display catalog apply stdout @@ -125,6 +105,8 @@ - name: clean up local temporary directory file: state: absent - path: "{{ local_temp.path}}" + path: "{{ inventory_dir }}/build/puppet/" delegate_to: 127.0.0.1 tags: puppet_fast + run_once: True + diff --git a/personal_infra/up.py b/personal_infra/up.py new file mode 100755 index 0000000..55d0f9b --- /dev/null +++ b/personal_infra/up.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +import argparse +from concurrent import futures +import pathlib +import shlex +import shutil +import subprocess +import textwrap +import yaml + +""" +directory/ + global_vars/*.json: these JSON files will be available to all hosts + host_vars/{host}/*.json: these JSON files will be available in each host + facts/{host}.json: output from "facter -y" for each host + +directory/ + output/ + {host}/ + catalog.json + modules/ +""" + + +def build_hiera(directory, build_host_dir, host): + hiera_data_dir = build_host_dir / "data" + hiera_data_dir.mkdir() + + hiera = { + "version": 5, + "hierarchy": [] + } + + global_vars_dir = directory / "global_vars" + + for global_var in global_vars_dir.glob("*.json"): + shutil.copy(global_var, hiera_data_dir / global_var.name) + hiera["hierarchy"].append({ + "name": global_var.name.removesuffix(".json"), + "path": global_var.name, + "data_hash": "json_data", + }) + + host_vars_dir = directory / "host_vars" / host + + for host_var in host_vars_dir.glob("*.json"): + shutil.copy(host_var, hiera_data_dir / host_var.name) + hiera["hierarchy"].append({ + "name": host_var.name.removesuffix(".json"), + "path": host_var.name, + "data_hash": "json_data", + }) + + hiera_path = build_host_dir / "hiera.yaml" + with open(hiera_path, "w") as f: + yaml.dump(hiera, f) + + return hiera_path + + +def build_facts(directory, build_host_dir, host): + source_facts_dir = directory / "facts" + + with open(source_facts_dir / f"{host}.yaml") as f: + facts_yaml_content = f.read() + + dest_facts_dir = build_host_dir / "yaml" / "facts" + dest_facts_dir.mkdir(parents=True) + + with open(dest_facts_dir / f"{host}.yaml", "w") as f: + f.write("--- !ruby/object:Puppet::Node::Facts\nvalues:\n") + f.write(textwrap.indent(facts_yaml_content, " ")) + + +def compile_catalog(directory, build_dir, modulepath, manifest, output_dir, + host): + build_host_dir = build_dir / host + build_host_dir.mkdir() + + hiera_path = build_hiera(directory, build_host_dir, host) + + build_facts(directory, build_host_dir, host) + + cmd = [ + "puppet", "catalog", "compile", + f"--modulepath={modulepath}", + f"--hiera_config={hiera_path}", + f"--manifest={manifest}", + "--terminus", "compiler", + "--vardir", build_host_dir, + "--facts_terminus", "yaml", + host + ] + print(shlex.join(map(str, cmd))) + catalog_compile = subprocess.run( + cmd, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf8" + ) + assert not catalog_compile.stderr, catalog_compile.stderr + + catalog_stdout = catalog_compile.stdout + + _, catalog = catalog_stdout.split("\n", 1) + + host_output_dir = output_dir / host + host_output_dir.mkdir() + with open(host_output_dir / "catalog.json", "w") as f: + f.write(catalog) + + shutil.copytree(modulepath, host_output_dir / "modules") + + +def up(directory: pathlib.Path, modulepath, manifest, hosts: list[str]): + build_dir = directory / "build" + build_dir.mkdir() + + output_dir = build_dir / "output" + output_dir.mkdir() + + def _compile_catalog(host): + compile_catalog( + directory=directory, + build_dir=build_dir, + modulepath=modulepath, + manifest=manifest, + output_dir=output_dir, + host=host) + + # list because exceptions do not happen unless you iterate over the result + list(futures.ThreadPoolExecutor().map(_compile_catalog, hosts)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory") + parser.add_argument("modulepath") + parser.add_argument("manifest") + parser.add_argument("hosts", nargs="+", metavar="host") + + args = parser.parse_args() + up( + directory=pathlib.Path(args.directory), + modulepath=args.modulepath, + manifest=args.manifest, + hosts=args.hosts + ) + + +if __name__ == "__main__": + main() -- 2.47.3