ext/pillar/gitstack.py

@@ -0,0 +1,179 @@
+from __future__ import absolute_import
+
+# Import python libs
+import copy
+import logging
+import os
+
+# Import salt libs
+import salt.loader
+import salt.utils
+import salt.utils.gitfs
+import salt.utils.dictupdate
+import salt.pillar.git_pillar
+
+# Import third party libs
+import salt.ext.six as six
+
+try:
+    from salt.utils.data import repack_dictlist
+except ImportError:
+    from salt.utils import repack_dictlist
+
+PER_REMOTE_OVERRIDES = salt.pillar.git_pillar.PER_REMOTE_OVERRIDES
+PER_REMOTE_ONLY = tuple(set(list(salt.pillar.git_pillar.PER_REMOTE_ONLY) + ['stack']))
+
+# Set up logging
+log = logging.getLogger(__name__)
+
+# Define the module's virtual name
+__virtualname__ = 'gitstack'
+
+
+def __virtual__():
+    '''
+    Only load if gitstack pillars are defined
+    '''
+    gitstack_pillars = [x for x in __opts__['ext_pillar'] if 'gitstack' in x]
+    if not gitstack_pillars:
+        # No gitstack external pillars were configured
+        return False
+
+    return __virtualname__
+
+
+def ext_pillar(minion_id, pillar, *repos, **single_repo_conf):
+
+    # Checkout the ext_pillar sources
+    opts = copy.deepcopy(__opts__)
+    opts['pillar_roots'] = {}
+    opts['__git_pillar'] = True
+    stacks = []
+    invalid_repos_idx = []
+
+    ## legacy configuration with a plain dict under gitstack ext_pillar key
+    if single_repo_conf and single_repo_conf.get('repo', None) is not None:
+        branch = single_repo_conf.get('branch', 'master')
+        repo = single_repo_conf['repo']
+        remote = ' '.join([branch, repo])
+        init_gitpillar_args = [ [remote], PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY ]
+        if 'stack' not in single_repo_conf:
+            log.error('A stack key is mandatory in gitstack configuration')
+            return {}
+
+    ## new configuration way
+    elif isinstance(repos, (list, tuple)) and len(repos) > 0:
+        for repo_idx, repo in enumerate(repos):
+            kw = repack_dictlist(repo[next(iter(repo))])
+            if 'stack' not in kw:
+                # stack param is mandatory in gitstack repos configuration
+                log.warning('Configuration for gitstack must contain a stack key for each repo.')
+                log.warning('Configured gitstack repo %s (at position %d) will be ignored' % (next(iter(repo)), repo_idx))
+                invalid_repos_idx.append(repo_idx)
+                continue
+
+            stacks.append(kw['stack'])
+
+        valid_repos = [repo for repo_idx, repo in enumerate(repos) if repo_idx not in invalid_repos_idx]
+        init_gitpillar_args = [ valid_repos, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY ]
+
+    else:
+        ### Invalid configuration
+        log.error('Configuration for gitstack must be a list of dicts or a single dict')
+        return {}
+
+    # check arguments to use with GitPillar, we could check also salt version
+    if len(_get_function_varnames(salt.utils.gitfs.GitPillar.__init__)) > 2:
+        # Include GLOBAL_ONLY args for Salt versions that require it
+        if 'global_only' in _get_function_varnames(salt.utils.gitfs.GitPillar.__init__):
+            init_gitpillar_args.append(salt.pillar.git_pillar.GLOBAL_ONLY)
+
+        # Initialize GitPillar object
+        gitpillar = salt.utils.gitfs.GitPillar(opts, *init_gitpillar_args)
+
+    else:
+        # Include GLOBAL_ONLY args for Salt versions that require it
+        if 'global_only' in _get_function_varnames(salt.utils.gitfs.GitPillar.init_remotes):
+            init_gitpillar_args.append(salt.pillar.git_pillar.GLOBAL_ONLY)
+
+        # Initialize GitPillar object
+        gitpillar = salt.utils.gitfs.GitPillar(opts)
+        gitpillar.init_remotes(*init_gitpillar_args)
+
+
+    if __opts__.get('__role') == 'minion':
+        # If masterless, fetch the remotes. We'll need to remove this once
+        # we make the minion daemon able to run standalone.
+        gitpillar.fetch_remotes()
+    gitpillar.checkout()
+
+    # Prepend the local path of the cloned Git repo
+    if not gitpillar.pillar_dirs:
+        log.error('Repositories used by gitstack must be included in the git pillar configuration')
+        return {}
+
+    # Initialize variables
+    stack_config = []
+    stack_config_kwargs = {}
+
+    # Replace relative paths with the absolute path of the cloned repository
+    if single_repo_conf:
+        stack_config = _resolve_stack(single_repo_conf['stack'], list(gitpillar.pillar_dirs.items())[0][0])
+    else:
+        pillar_dirs = list(gitpillar.pillar_dirs.keys())
+        for idx, stack in enumerate(stacks):
+            try:
+                pillar_dir = pillar_dirs[idx]
+            except IndexError:
+                log.warning('Ignoring gitstack stack configuration: %s' % stack)
+                log.warning('Ignoring gitstack repo maybe failed checkout')
+                continue
+
+            if isinstance(stack, dict):
+                # TODO: use salt.utils.dictupdate.merge
+                stack_config_kwargs.update(_resolve_stack(stack, pillar_dir))
+            elif isinstance(stack, list):
+                stack_config.extend(_resolve_stack(stack, pillar_dir))
+            else:
+                stack_config.append(_resolve_stack(stack, pillar_dir))
+
+    # Load the 'stack' pillar module
+    stack_pillar = salt.loader.pillars(__opts__, __salt__, __context__)['stack']
+
+    # Call the 'stack' pillar module
+    if isinstance(stack_config, dict):
+        return stack_pillar(minion_id, pillar, **stack_config)
+
+    elif isinstance(stack_config, list):
+        return stack_pillar(minion_id, pillar, *stack_config, **stack_config_kwargs)
+
+    else:
+        return stack_pillar(minion_id, pillar, stack_config)
+
+
+def _resolve_stack(x, path):
+    '''
+    Resolve relative paths to the absolute path of the cloned Git repo
+    '''
+    if isinstance(x, dict):
+        y = {}
+        for key, value in six.iteritems(x):
+            y[key] = _resolve_stack(value, path)
+    elif isinstance(x, list):
+        y = []
+        for item in x:
+            y.append(_resolve_stack(item, path))
+    elif isinstance(x, six.string_types):
+        y = os.path.join(path, x)
+    else:
+        y = x
+    return y
+
+
+def _get_function_varnames(fn):
+    '''
+    Return the var names for a function
+    '''
+    if six.PY2:
+        return fn.im_func.func_code.co_varnames
+    return fn.__code__.co_varnames

pillar/base.sls

@@ -0,0 +1,2 @@
+some-key: some-value
+what-is-the-saltenv: {{ saltenv }}

pillar/top.sls

@@ -0,0 +1,3 @@
+{{ saltenv }}:
+  '*':
+  - base

stack/file.sls

@@ -0,0 +1,2 @@
+stack-env: {{ saltenv }}
+stack-value: this is from pillar stack, from gitfs!

stack/stack.cfg

@@ -0,0 +1 @@
+- file.sls

states/salt/minion/init.sls

@@ -1,40 +0,0 @@
-{%- set masters = ['salt.nerdhouse.io'] %}
-
-salt-minion:
-  pkg.installed:
-  - version: latest
-  service.running:
-  - enable: true
-  - require:
-    - pkg: salt-minion
-    - file: /etc/salt/minion
-
-restart-salt-minion:
-  cmd.run:
-  - name: sleep 10 && systemctl restart salt-minion
-  - bg: true
-  - order: last
-  - onchanges:
-    - pkg: salt-minion
-    - file: /etc/salt/minion
-  - require:
-    - cmd: check-minion-config
-
-/etc/salt/minion:
-  file.managed:
-  - source: salt://{{ slspath }}/templates/minion.jinja
-  - template: jinja
-  - user: root
-  - group: root
-  - mode: 644
-  - context:
-      masters: {{ masters|json }}
-  - require:
-    - pkg: salt-minion
-
-check-minion-config:
-  cmd.run:
-  - name: sudo salt-call --local --skip-grains test.true
-  - onchanges:
-    - pkg: salt-minion
-    - file: /etc/salt/minion

states/salt/minion/templates/minion.jinja

@@ -1,26 +0,0 @@
-id: {{ grains.id|lower }}
-log_level: info
-ipv6: true
-ssl: true
-transport: tcp
-
-# https://github.com/saltstack/salt/blob/b95213ec903402f25c1e0aeb3990fe8452ab63ce/conf/minion#L39-L47
-# str, failover or disable
-master_type: str
-{% if masters is defined %}
-master:
-  {%- for master in masters %}
-- {{ master }}
-  {%- endfor %}
-{%- endif %}
-
-# set the default saltenv for highstate
-# otherwise it tries to run all of them 🤔
-saltenv: base
-# useful for influencing git_pillar and gitfs at the same time
-pillarenv_from_saltenv: true
-
-enable_fqdns_grains: false
-enable_gpu_grains: false
-
-# vim: ft=yaml

states/sshd/init.sls

@@ -1,6 +1,3 @@
-include:
-- users
-
 openssh-client:
   pkg.installed:
   - version: latest
@@ -9,6 +6,3 @@ sshd-service:
   service.running:
   - name: ssh
   - enable: true
-  - require:
-    - sls: users
-    - pkg: openssh-client

states/top.sls

@@ -1,5 +1,3 @@
 {{ saltenv }}:
   '*':
-  - users
   - sshd
-  - salt.minion

states/users/init.sls

@@ -1,24 +0,0 @@
-{%- import_yaml 'users/users.jinja' as users %}
-
-{%- if users is not mapping %}
-  {%- do raise('users/users.jinja is malformed') %}
-{%- endif %}
-
-{%- for name, user in users.items() %}
-{{ name|json }}:
-  user.present:
-  - usergroup: true
-  - createhome: true
-  - groups: {{ user.get('groups', [])|json }}
-  {%- if user.password is defined %}
-  - password: {{ user.password|json }}
-  {%- endif %}
-  {%- if user.shell is defined %}
-  - shell: {{ user.shell }}
-  {%- endif %}
-  ssh_auth.manage:
-  - user: {{ name }}
-  {%- if user['ssh-keys'] is defined %}
-  - ssh_keys: {{ user['ssh-keys']|json }}
-  {%- endif %}
-{%- endfor %}

states/users/users.jinja

@@ -1,29 +0,0 @@
-{%- set sudo_group = 'sudo' if grains.os|lower == 'debian' else 'wheel' %}
-{%- set admin_groups = [sudo_group] %}
-{%- if grains.init == 'systemd' %}
-  {%- do admin_groups.append('systemd-journal') %}
-{%- endif %}
-
-{# FIXME(frebib): manage Docker group instead of blindly adding it #}
-{%- do admin_groups.append('docker') %}
-
-frebib:
-  groups:
-{%- for group in admin_groups %}
-  - {{ group }}
-{%- endfor %}
-  ssh-keys:
-  - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINk+sOUEXKsGqITyMhna9v77ADGagkr3YMpgZFkrvqcd frebib@frebib-PC
-  - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBIGxhZPQM/3Ck+DNNM0CoIZTsvKqQLKq8fqQoO6fXzX frebib@frebib-OnePlus3
-  - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+T8ChEU9YmpE2BY77oEtKzedB8HWDSM5bErDN9gcvj frebib@frebib-Cf
-
-adam:
-  groups:
-{%- for group in admin_groups %}
-  - {{ group }}
-{%- endfor %}
-  ssh-keys:
-  - ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAiF/WxxMOOE/r8I/anF8gKarjNFmeoPuXNMYE0Hwo/NMqbJ9qKVONn+4fa5T99yhhAnYy92PBPxjmSsOqex28XZFh4I4GBGKh+Su1tJKLXsts1rbJmB1gSpKGbbjsiZcta4FqbiLKyUGL1wiV6GWucKdkC9lfTfocmH20tLbqSTryTAfy62oaEyEUSSATcwnl6ITF6BaQVGT8e78O4DpMYvYXFDa0nre0GlBigI0bAIHknmo+5JuL9i6RbnLr/zHMBiT3lnqQ8IO0JLNJD3ML/X/vgo3Htd6ovKe28YBi+LV+olpcFlQGSpNj+gE67UYb+Hw4mMTjzG4RhNHfQZCrNw== adampc
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYo04d3a7J22F0NlAM1MeqmZrbF7iaF9p9whLQmq/gYgZnFKtW7G/rQCUUG5X8yEn5LXkc9SYdRceyaleNDAe0/uEJIbbm+VPgkShV+pyA8AbP4+OF95JKgCS6/B59IStF8GrVnX9krjYLYVwS3FqwvYt/ZkYriX4hkaTkf3FslLU7eEwrDec3RuGXhTDTdLdFyq6wXsI/bITX2C0b8D5obNNkyPJFVV1zshy9mQ5/5wjHYLyZhUSRzcY6xDOlxWmTSHseYG1U5espgba7t6ZaL/n44IrXzFsuXRIA74rx3ESAtCEesM3TaPI/Q8dHOYsCdDquTyccP/nEkDCxxhl3 adam-laptop
-  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1V9+YmjPyOVdN4tj0MvyyZ4ZMZWkkTeQqEmRRNN0JcogunG97vo32UiZnt2gSPLZjutK5ql1SGTdgbaoSU4ci5vkKjhB8FpozupzvIy2qZ35v2K3eMsU43bLNY23R+ixDpXWkDgXQSVXTZdK3v/zNtDU3RE8vOqa9GFNl9DCGr+ppGfGxWOR6SMcuHu26dZf8pges3D23465kVzWJre3I7G3AGo49y0b9XCk1SkQPhPqQABcSv+k3Cp7hGdvSpj2kS29FRyHZZ5fksAX6N/Csz5Pc/4FyGY4S8Kd0SwdvxNVAGmWPIoB6cg+PBYnsSRYLUcSkWNpEkxK21u3kIS6F02NRs8kbPznr5S+S1E6PHmMRvCfmvegnh+QF+/W88kb9LBaicgDCo+A3vcLC5pfMPRFEXXNkKltlXDkO7/wGC93pUrP0X0sx9QuyqwAPacnZd0OX4uX7jJeMsbGSq+tXLGp1kGOET4J9GeoQrIK/NUxQkF6zDhWH6FcbescmIma4Ys3c1XZHs19DysVC7U/0wr8USzm2ljQ65Nyp0Hv18DdZZerQHkFx0LCJedrVaPC87xBHZORQlrvaF307dYaRLzCn6Jf9dYM2HgK/F0Y7hZv5KhU6J7upXivUJFN6/VMGCptA9DFLSje5Y7vXeWZkTEfghbm2Ylk2BtzuT7CLYw== adam-cpwc
-