Port Puppet compiler to Python

diff --git a/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml b/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml
index 0ee291ef6d959f2331341ced5ef5c2d13d9e8885..3512717ee2763185fa46cc320d97892cfe6d455b 100644 (file)
--- a/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml
+++ b/personal_infra/playbooks/roles/apply_puppet/tasks/main.yml
@@ -1,97 +1,77 @@
 ---
-- name: create local temporary directory
-  tempfile:
-    state: directory
-    path: "{{ inventory_dir }}/tmp"
-  register: local_temp
-  delegate_to: 127.0.0.1
+- name: clean puppet build directory
+  local_action:
+    module: file
+    path: "{{ inventory_dir }}/build/puppet"
+    state: absent
+  run_once: True
   tags: puppet_fast
-- name: create data directory in local temp
-  file:
-    path: "{{ local_temp.path }}/data"
+- name: create puppet build directories
+  local_action:
+    module: file
+    path: "{{ inventory_dir }}/{{ item }}"
+    state: directory
+  loop:
+    - build/puppet/global_vars
+    - build/puppet/host_vars
+    - build/puppet/facts
+  run_once: True
+  tags: puppet_fast
+- name: create puppet build host vars directories
+  local_action:
+    module: file
+    path: "{{ inventory_dir }}/build/puppet/host_vars/{{ inventory_hostname }}"
     state: directory
-  delegate_to: 127.0.0.1
-  tags: puppet_fast
-- name: create hiera.yaml
-  copy:
-    dest: "{{ local_temp.path }}/hiera.yaml"
-    content: |
-      version: 5
-      hierarchy:
-        - name: hostvars
-          path: hostvars.json
-          data_hash: json_data
-        - name: this
-          path: this.json
-          data_hash: json_data
-  delegate_to: 127.0.0.1
   tags: puppet_fast
 - name: dump hostvars
-  copy:
-    dest: "{{ local_temp.path }}/data/hostvars.json"
+  local_action:
+    module: copy
+    dest: "{{ inventory_dir }}/build/puppet/global_vars/hostvars.json"
     content: "{'hostvars': {{ hostvars }} }"
-  delegate_to: 127.0.0.1
+  run_once: True
   tags: puppet_fast
 - name: dump this
-  copy:
-    dest: "{{ local_temp.path }}/data/this.json"
+  local_action:
+    module: copy
+    dest: "{{ inventory_dir }}/build/puppet/host_vars/{{ inventory_hostname }}/this.json"
     content: "{{ hostvars[inventory_hostname] }}"
-  delegate_to: 127.0.0.1
   tags: puppet_fast
-- name: install epel
-  package:
-    name: epel-release
-  when: ansible_distribution_file_variety == 'RedHat'
-- name: install packages
-  package:
-    name:
-      - puppet
-      - unzip
 - name: get facts
   command: facter -y
   register: facter_output
   tags: puppet_fast
-- name: create facts directory in local temp
-  file:
-    path: "{{ local_temp.path }}/yaml/facts"
-    state: directory
-  delegate_to: 127.0.0.1
-  tags: puppet_fast
 - name: dump facts
-  copy:
-    dest: "{{ local_temp.path }}/yaml/facts/{{ inventory_hostname }}.yaml"
-    content: "--- !ruby/object:Puppet::Node::Facts\nvalues:\n  {{ facter_output.stdout | indent(width=2) }}"
+  local_action:
+    module: copy
+    dest: "{{ inventory_dir }}/build/puppet/facts/{{ inventory_hostname }}.yaml"
+    content: "{{ facter_output.stdout }}"
   delegate_to: 127.0.0.1
   tags: puppet_fast
-- name: compile catalogs
-  command: puppet catalog compile --modulepath={{ inventory_dir }}/puppet/modules --hiera_config={{ local_temp.path }}/hiera.yaml --manifest={{ inventory_dir }}/puppet/site --terminus compiler --vardir {{ local_temp.path }}/ --facts_terminus yaml {{ inventory_hostname }}
+- name: compile puppet catalogs
+  local_action:
+    module: command
+    cmd: "{{ inventory_dir }}/up.py {{ inventory_dir }}/build/puppet {{ inventory_dir }}/puppet/modules {{ inventory_dir }}/puppet/site {% for host in ansible_play_batch %}{{ host }} {% endfor %}"
+  tags: puppet_fast
+  run_once: True
+- name: package catalog
+  archive:
+    path: "{{ inventory_dir }}/build/puppet/build/output/{{ inventory_hostname }}"
+    dest: "{{ inventory_dir }}/build/puppet/puppet_catalog_{{ inventory_hostname }}.zip"
+    format: zip
   delegate_to: 127.0.0.1
-  register: catalog
   tags: puppet_fast
 - name: create remote temporary directory
   tempfile:
     state: directory
   register: remote_temp
   tags: puppet_fast
-- name: write catalog
-  copy:
-    dest: "{{ remote_temp.path }}/catalog.json"
-    content: "{{ catalog.stdout | regex_replace('\\A.*?\\n', multiline=True) }}"
-  tags: puppet_fast
-- name: package modules
-  archive:
-    path: ../puppet/modules
-    dest: "{{ local_temp.path }}/puppet_modules.zip"
-    format: zip
-  delegate_to: 127.0.0.1
-  tags: puppet_fast
-- name: unpackage modules
+- name: unpackage catalog
   unarchive:
-    src: "{{ local_temp.path }}/puppet_modules.zip"
+    src: "{{ inventory_dir }}/build/puppet/puppet_catalog_{{ inventory_hostname }}.zip"
     dest: "{{ remote_temp.path }}"
   tags: puppet_fast
 - name: preview catalog
-  command: puppet apply --catalog {{ remote_temp.path }}/catalog.json --noop --test --modulepath={{ remote_temp.path }}/modules/
+  command: puppet apply --catalog {{ remote_temp.path }}/{{ inventory_hostname }}/catalog.json --noop --test --modulepath={{ remote_temp.path }}/{{ inventory_hostname }}/modules/
   register: catalog_apply
   tags: puppet_fast
 - name: display catalog preview stdout
@@ -106,7 +86,7 @@
   pause:
   tags: pause
 - name: apply catalog
-  command: puppet apply --catalog {{ remote_temp.path }}/catalog.json --modulepath={{ remote_temp.path }}/modules/
+  command: puppet apply --catalog {{ remote_temp.path }}/{{ inventory_hostname }}/catalog.json --modulepath={{ remote_temp.path }}/{{ inventory_hostname }}/modules/
   register: catalog_apply
   tags: puppet_fast
 - name: display catalog apply stdout
@@ -125,6 +105,8 @@
 - name: clean up local temporary directory
   file:
     state: absent
-    path: "{{ local_temp.path}}"
+    path: "{{ inventory_dir }}/build/puppet/"
   delegate_to: 127.0.0.1
   tags: puppet_fast
+  run_once: True
+

diff --git a/personal_infra/up.py b/personal_infra/up.py
new file mode 100755 (executable)
index 0000000..55d0f9b
--- /dev/null
+++ b/personal_infra/up.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+import argparse
+from concurrent import futures
+import pathlib
+import shlex
+import shutil
+import subprocess
+import textwrap
+import yaml
+
+"""
+directory/
+  global_vars/*.json: these JSON files will be available to all hosts
+  host_vars/{host}/*.json: these JSON files will be available in each host
+  facts/{host}.json: output from "facter -y" for each host
+
+directory/
+  output/
+    {host}/
+      catalog.json
+      modules/
+"""
+
+
+def build_hiera(directory, build_host_dir, host):
+    hiera_data_dir = build_host_dir / "data"
+    hiera_data_dir.mkdir()
+
+    hiera = {
+        "version": 5,
+        "hierarchy": []
+    }
+
+    global_vars_dir = directory / "global_vars"
+
+    for global_var in global_vars_dir.glob("*.json"):
+        shutil.copy(global_var, hiera_data_dir / global_var.name)
+        hiera["hierarchy"].append({
+            "name": global_var.name.removesuffix(".json"),
+            "path": global_var.name,
+            "data_hash": "json_data",
+        })
+
+    host_vars_dir = directory / "host_vars" / host
+
+    for host_var in host_vars_dir.glob("*.json"):
+        shutil.copy(host_var, hiera_data_dir / host_var.name)
+        hiera["hierarchy"].append({
+            "name": host_var.name.removesuffix(".json"),
+            "path": host_var.name,
+            "data_hash": "json_data",
+        })
+
+    hiera_path = build_host_dir / "hiera.yaml"
+    with open(hiera_path, "w") as f:
+        yaml.dump(hiera, f)
+
+    return hiera_path
+
+
+def build_facts(directory, build_host_dir, host):
+    source_facts_dir = directory / "facts"
+
+    with open(source_facts_dir / f"{host}.yaml") as f:
+        facts_yaml_content = f.read()
+
+    dest_facts_dir = build_host_dir / "yaml" / "facts"
+    dest_facts_dir.mkdir(parents=True)
+
+    with open(dest_facts_dir / f"{host}.yaml", "w") as f:
+        f.write("--- !ruby/object:Puppet::Node::Facts\nvalues:\n")
+        f.write(textwrap.indent(facts_yaml_content, "  "))
+
+
+def compile_catalog(directory, build_dir, modulepath, manifest, output_dir,
+                    host):
+    build_host_dir = build_dir / host
+    build_host_dir.mkdir()
+
+    hiera_path = build_hiera(directory, build_host_dir, host)
+
+    build_facts(directory, build_host_dir, host)
+
+    cmd = [
+        "puppet", "catalog", "compile",
+        f"--modulepath={modulepath}",
+        f"--hiera_config={hiera_path}",
+        f"--manifest={manifest}",
+        "--terminus", "compiler",
+        "--vardir", build_host_dir,
+        "--facts_terminus", "yaml",
+        host
+    ]
+    print(shlex.join(map(str, cmd)))
+    catalog_compile = subprocess.run(
+        cmd, check=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf8"
+    )
+    assert not catalog_compile.stderr, catalog_compile.stderr
+
+    catalog_stdout = catalog_compile.stdout
+
+    _, catalog = catalog_stdout.split("\n", 1)
+
+    host_output_dir = output_dir / host
+    host_output_dir.mkdir()
+    with open(host_output_dir / "catalog.json", "w") as f:
+        f.write(catalog)
+
+    shutil.copytree(modulepath, host_output_dir / "modules")
+
+
+def up(directory: pathlib.Path, modulepath, manifest, hosts: list[str]):
+    build_dir = directory / "build"
+    build_dir.mkdir()
+
+    output_dir = build_dir / "output"
+    output_dir.mkdir()
+
+    def _compile_catalog(host):
+        compile_catalog(
+            directory=directory,
+            build_dir=build_dir,
+            modulepath=modulepath,
+            manifest=manifest,
+            output_dir=output_dir,
+            host=host)
+
+    # list because exceptions do not happen unless you iterate over the result
+    list(futures.ThreadPoolExecutor().map(_compile_catalog, hosts))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("directory")
+    parser.add_argument("modulepath")
+    parser.add_argument("manifest")
+    parser.add_argument("hosts", nargs="+", metavar="host")
+
+    args = parser.parse_args()
+    up(
+        directory=pathlib.Path(args.directory),
+        modulepath=args.modulepath,
+        manifest=args.manifest,
+        hosts=args.hosts
+    )
+
+
+if __name__ == "__main__":
+    main()