From f4326afd766daaec60da7fa340b3e28d21d98365 Mon Sep 17 00:00:00 2001
From: Bombar Maxime <bombar@crans.org>
Date: Tue, 28 Apr 2020 22:29:12 +0200
Subject: [PATCH] [re2o_lookup] Make use of cache.
---
ansible.cfg | 7 +
lookup_plugins/re2oapi.py | 287 ++++++++++++++++++++++++++++----------
2 files changed, 224 insertions(+), 70 deletions(-)
diff --git a/ansible.cfg b/ansible.cfg
index ec5d521e..5b23c72b 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -45,3 +45,10 @@ api_hostname = intranet.crans.org
# Whether or not using vault_cranspasswords
use_cpasswords = True
+
+# Specify cache plugin for re2o API. By default, cache nothing
+cache = jsonfile
+
+# Time in second before the cache expired. 0 means never expire cache.
+# Default is 120 seconds.
+timeout = 120
diff --git a/lookup_plugins/re2oapi.py b/lookup_plugins/re2oapi.py
index 9099c9e3..53d23555 100644
--- a/lookup_plugins/re2oapi.py
+++ b/lookup_plugins/re2oapi.py
@@ -7,6 +7,8 @@ For a detailed example look at https://github.com/ansible/ansible/blob/3dbf89e8a
The API Client has been adapted from https://gitlab.federez.net/re2o/re2oapi
"""
+from ansible.plugins.loader import cache_loader
+
from pathlib import Path
import datetime
import requests
@@ -340,6 +342,73 @@ class LookupModule(LookupBase):
- debug: var=dnszones
"""
+ def _readconfig(self, section="re2o", key=None, boolean=False,
+ integer=False):
+ config = self._config
+ if not config:
+ return None
+ else:
+ if config.has_option(section, key):
+ display.vvv("Found key {} in configuration file".format(key))
+ if boolean:
+ return config.getboolean(section, key)
+ elif integer:
+ return config.getint(section, key)
+ else:
+ return config.get(section, key)
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ config_manager = ConfigManager()
+ config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
+ self._config = ConfigParser()
+ self._config.read(config_file)
+
+ display.vvv("Using {} as configuration file.".format(config_file))
+
+ self._api_hostname = None
+ self._api_username = None
+ self._api_password = None
+ self._use_cpasswords = None
+ self._cache_plugin = None
+ self._cache = None
+ self._timeout = 120
+
+ if self._config.has_section("re2o"):
+ display.vvv("Found section re2o in configuration file")
+
+ self._api_hostname = self._readconfig(key="api_hostname")
+ self._use_cpasswords = self._readconfig(key="use_cpasswords",
+ boolean=True)
+ self._cache_plugin = self._readconfig(key="cache")
+ self._timeout = self._readconfig(key="timeout", integer=True)
+
+ if self._cache_plugin is not None:
+ display.vvv("Using {} as cache plugin".format(self._cache_plugin))
+
+ if self._cache_plugin == 'jsonfile':
+ self._cachedir = Path.home() / ".cache/Ansible/re2oapi"
+ display.vvv("Cache directory is {}".format(self._cachedir))
+ if not self._cachedir.exists():
+ # Creates Ansible cache directory with right permissions
+ # if it doesn't exist yet.
+ display.vvv("Cache directory doesn't exist. Creating it.")
+ try:
+ self._cachedir.mkdir(mode=0o700, parents=True)
+ except Exception as e:
+ raise AnsibleError("""Unable to create {dir}.
+ Original error was : {err}"""
+ .format(dir=self._cachedir,
+ err=to_native(e)))
+ self._cache = cache_loader.get('jsonfile',
+ _uri=self._cachedir,
+ _timeout=self._timeout,
+ )
+ else:
+ raise AnsibleError("Cache plugin {} not supported"
+ .format(self._cache_plugin))
+
def run(self, terms, variables=None, api_hostname=None, api_username=None,
api_password=None, use_tls=True):
@@ -354,33 +423,20 @@ class LookupModule(LookupBase):
:returns: A list of results to the specific queries.
"""
- config_manager = ConfigManager()
- config_file = config_manager.data.get_setting(name="CONFIG_FILE").value
- config = ConfigParser()
- config.read(config_file)
-
- use_cpasswords = False
+ # Use the hostname specified by the user if it exists.
+ if api_hostname is not None:
+ display.vvv("Overriding api_hostname with {}".format(api_hostname))
+ else:
+ api_hostname = self._api_hostname
- if config.has_section("re2o"):
- display.vvv("Found section re2o in configuration file")
- if config.has_option("re2o", "api_hostname"):
- display.vvv("Found option api_hostname in config file")
- api_hostname = config.get("re2o", "api_hostname")
- display.vvv("Override api_hostname with {} from configuration"
- .format(api_hostname))
- if config.has_option("re2o", "use_cpasswords"):
- display.vvv("Found option use_cpasswords in config file")
- use_cpasswords = config.getboolean("re2o", "use_cpasswords")
- display.vvv("Override api_hostname with {} from configuration"
- .format(use_cpasswords))
-
- if api_hostname is None:
+ if self._api_hostname is None:
raise AnsibleError(to_native(
'You must specify a hostname to contact re2oAPI'
))
- if api_username is None and api_password is None and use_cpasswords:
- display.vvv("Use cpasswords vault to get API credentials.")
+ if (api_username is None and api_password is None
+ and self._use_cpasswords):
+ display.vvv("Using cpasswords vault to get API credentials.")
api_username = variables.get('vault_re2o_service_user')
api_password = variables.get('vault_re2o_service_password')
@@ -399,7 +455,7 @@ class LookupModule(LookupBase):
res = []
dterms = collections.deque(terms)
- machines_roles = None # TODO : Cache this.
+
display.vvv("Lookup terms are {}".format(terms))
while dterms:
term = dterms.popleft()
@@ -411,10 +467,7 @@ class LookupModule(LookupBase):
elif term == 'get_role':
try:
role_name = dterms.popleft()
- roles, machines_roles = self._get_role(api_client,
- role_name,
- machines_roles,
- )
+ roles = self._get_role(api_client, role_name)
res.append(roles)
except IndexError:
display.v("Error in re2oapi : No role_name provided")
@@ -429,59 +482,153 @@ class LookupModule(LookupBase):
.format(to_native(e)))
return res
+ def _get_cache(self, key):
+ if self._cache:
+ return self._cache.get(key)
+ else:
+ return None
+
+ def _set_cache(self, key, value):
+ if self._cache:
+ return self._cache.set(key, value)
+ else:
+ return None
+
+ def _is_cached(self, key):
+ if self._cache:
+ return self._cache.contains(key)
+ else:
+ return False
+
def _getzones(self, api_client):
display.v("Getting dns zone names")
- zones = api_client.list('dns/zones')
- zones_name = [zone["name"][1:] for zone in zones]
+ zones, zones_name = None, None
+
+ if self._is_cached('dnszones'):
+ zones_name = self._get_cache('dnszones')
+
+ if zones_name is not None:
+ display.vvv("Found dnszones in cache.")
+
+ else:
+ if self._is_cached('dns_zones'):
+ zones = self._get_cache('dns_zones')
+ if zones is not None:
+ display.vvv("Found dns/zones in cache.")
+ else:
+ display.vvv("Contacting the API, endpoint dns/zones...")
+ zones = api_client.list('dns/zones')
+ display.vvv("...Done")
+ zones_name = [zone["name"][1:] for zone in zones]
+ display.vvv("Storing dnszones in cache.")
+ self._set_cache('dnszones', zones_name)
+
return zones_name
def _getreverse(self, api_client):
display.v("Getting dns reverse zones")
- display.vvv("Contacting the API, endpoint dns/reverse-zones...")
- zones = api_client.list('dns/reverse-zones')
- display.vvv("...Done")
- res = []
- for zone in zones:
- if zone['ptr_records']:
- display.vvv('Found PTR records')
- subnets = []
- for net in zone['cidrs']:
- net = netaddr.IPNetwork(net)
- if net.prefixlen > 24:
- subnets.extend(net.subnet(32))
- elif net.prefixlen > 16:
- subnets.extend(net.subnet(24))
- elif net.prefixlen > 8:
- subnets.extend(net.subnet(16))
- else:
- subnets.extend(net.subnet(8))
- for subnet in subnets:
- _address = netaddr.IPAddress(subnet.first)
- rev_dns_a = _address.reverse_dns.split('.')[:-1]
- if subnet.prefixlen == 8:
- zone_name = '.'.join(rev_dns_a[3:])
- elif subnet.prefixlen == 16:
- zone_name = '.'.join(rev_dns_a[2:])
- elif subnet.prefixlen == 24:
- zone_name = '.'.join(rev_dns_a[1:])
- res.append(zone_name)
- display.vvv("Found reverse zone {}".format(zone_name))
+
+ zones, res = None, None
+
+ if self._is_cached('dnsreverse'):
+ res = self._get_cache('dnsreverse')
+
+ if res is not None:
+ display.vvv("Found dnsreverse in cache.")
+
+ else:
+ if self._is_cached('dns_reverse-zones'):
+ zones = self._get_cache('dns_reverse-zones')
+
+ if zones is not None:
+ display.vvv("Found dns/reverse-zones in cache.")
+ else:
+ display.vvv("Contacting the API, endpoint dns/reverse-zones..")
+ zones = api_client.list('dns/reverse-zones')
+ display.vvv("...Done")
+
+ display.vvv("Trying to format dns reverse in a nice way.")
+ res = []
+ for zone in zones:
+ if zone['ptr_records']:
+ display.vvv('Found PTR records')
+ subnets = []
+ for net in zone['cidrs']:
+ net = netaddr.IPNetwork(net)
+ if net.prefixlen > 24:
+ subnets.extend(net.subnet(32))
+ elif net.prefixlen > 16:
+ subnets.extend(net.subnet(24))
+ elif net.prefixlen > 8:
+ subnets.extend(net.subnet(16))
+ else:
+ subnets.extend(net.subnet(8))
+
+ for subnet in subnets:
+ _address = netaddr.IPAddress(subnet.first)
+ rev_dns_a = _address.reverse_dns.split('.')[:-1]
+ if subnet.prefixlen == 8:
+ zone_name = '.'.join(rev_dns_a[3:])
+ elif subnet.prefixlen == 16:
+ zone_name = '.'.join(rev_dns_a[2:])
+ elif subnet.prefixlen == 24:
+ zone_name = '.'.join(rev_dns_a[1:])
+ res.append(zone_name)
+ display.vvv("Found reverse zone {}".format(zone_name))
+
if zone['ptr_v6_records']:
display.vvv("Found PTR v6 record")
- net = netaddr.IPNetwork(zone['prefix_v6']+'/'+str(zone['prefix_v6_length']))
- net_class = max(((net.prefixlen -1) // 4) +1, 1)
+ net = netaddr.IPNetwork(zone['prefix_v6']
+ + '/'
+ + str(zone['prefix_v6_length']))
+ net_class = max(((net.prefixlen - 1) // 4) + 1, 1)
zone6_name = ".".join(
- netaddr.IPAddress(net.first).reverse_dns.split('.')[32 - net_class:])[:-1]
+ netaddr.IPAddress(net.first)
+ .reverse_dns.split('.')[32 - net_class:])[:-1]
res.append(zone6_name)
display.vvv("Found reverse zone {}".format(zone6_name))
- return list(set(res))
+
+ display.vvv("Storing dns reverse zones in cache.")
+ self._set_cache('dnsreverse', list(set(res)))
+
+ return res
def _rawquery(self, api_client, endpoint):
- display.v("Make a raw query to endpoint {}".format(endpoint))
- return api_client.list(endpoint)
-
- def _get_role(self, api_client, role_name, machines_roles):
- if machines_roles is None:
- machines_roles = api_client.list("machines/role")
- return list(filter(lambda machine: machine["role_type"] == role_name,
- machines_roles)), machines_roles
+ res = None
+ if self._is_cached(endpoint.replace('/', '_')):
+ res = self._get_cache(endpoint.replace('/', '_'))
+ if res is not None:
+ display.vvv("Found {} in cache.".format(endpoint))
+ else:
+ display.v("Making a raw query {host}/api/{endpoint}"
+ .format(host=self.api_hostname, endpoint=endpoint))
+ res = api_client.list(endpoint)
+ display.vvv("Storing result in cache.")
+ self._set_cache(endpoint.replace('/', '_'), res)
+ return res
+
+ def _get_role(self, api_client, role_name):
+ res, machines_roles = None, None
+
+ if self._is_cached(role_name):
+ res = self._get_cache(role_name)
+
+ if res is not None:
+ display.vvv("Found {} in cache.".format(role_name))
+ else:
+ if self._is_cached("machines_role"):
+ machines_roles = self._get_cache("machines_role")
+
+ if machines_roles is not None:
+ display.vvv("Found machines/roles in cache.")
+ else:
+ machines_roles = api_client.list("machines/role")
+ display.vvv("Storing machines/role in cache.")
+ self._set_cache("machines_role", machines_roles)
+
+ res = list(filter(lambda m: m["role_type"] == role_name,
+ machines_roles))
+ display.vvv("Storing {} in cache.".format(role_name))
+ self._set_cache(role_name, res)
+
+ return res
--
GitLab