From 15cf56ba655b474367c7912856027896ff49e5f5 Mon Sep 17 00:00:00 2001
From: Bombar Maxime <bombar@crans.org>
Date: Sat, 25 Apr 2020 23:10:01 +0200
Subject: [PATCH] Standalone re2o api lookup plugin
---
.gitmodules | 3 -
lookup_plugins/re2oapi | 1 -
lookup_plugins/re2oapi.py | 386 ++++++++++++++++++++++++++++++++++++--
3 files changed, 367 insertions(+), 23 deletions(-)
delete mode 160000 lookup_plugins/re2oapi
diff --git a/.gitmodules b/.gitmodules
index a5cf29f9..59564548 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,6 +1,3 @@
[submodule "roles/re2o-mail-server/templates/re2o-services/mail-server/mail-aliases"]
path = roles/re2o-mail-server/templates/re2o-services/mail-server/mail-aliases
url = https://gitlab.crans.org/nounous/mail-aliases
-[submodule "re2o-re2oapi"]
- path = lookup_plugins/re2oapi
- url = git@gitlab.crans.org:nounous/re2o-re2oapi.git
diff --git a/lookup_plugins/re2oapi b/lookup_plugins/re2oapi
deleted file mode 160000
index 6565b92f..00000000
--- a/lookup_plugins/re2oapi
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 6565b92f3bfc13d02b95888ae021f5bd6f7ef317
diff --git a/lookup_plugins/re2oapi.py b/lookup_plugins/re2oapi.py
index b4a87fc7..6b4bef87 100644
--- a/lookup_plugins/re2oapi.py
+++ b/lookup_plugins/re2oapi.py
@@ -1,34 +1,368 @@
"""
-A Proof Of Concept of lookup plugin to query the re2o API.
+A lookup plugin to query the re2o API.
For a detailed example look at https://github.com/ansible/ansible/blob/3dbf89e8aeb80eb2d1484b1cb63458e4bb12795a/lib/ansible/plugins/lookup/aws_ssm.py
-For now:
-
- - Need to clone nounous/re2o-re2oapi.git and checkout to crans branch.
- - This Re2oAPIClient needs python3-iso8601
-
-TODO: Implement a small client for our needs, this will also remove the sys.path extension ...
+The API Client has been adapted from https://gitlab.federez.net/re2o/re2oapi
"""
+from pathlib import Path
+import datetime
+import requests
+import stat
+import json
+
+from ansible.module_utils._text import to_native
from ansible.plugins.lookup import LookupBase
-from ansible.errors import AnsibleError
+from ansible.errors import (AnsibleError,
+ AnsibleFileNotFound,
+ AnsibleLookupError,
+ )
+from ansible.utils.display import Display
+
+# Ansible Logger to stdout
+display = Display()
+
+# Number of seconds before expiration where renewing the token is done
+TIME_FOR_RENEW = 120
+# Default name of the file to store tokens. Path $HOME/{DEFAUlt_TOKEN_FILENAME}
+DEFAULT_TOKEN_FILENAME = '.re2o.token'
+
+
+class Client:
+ """
+ Class based client to contact re2o API.
+ """
+ def __init__(self, hostname, username, password, use_tls=True):
+ """
+ :arg hostname: The hostname of the Re2o instance to use.
+ :arg username: The username to use.
+ :arg password: The password to use.
+ :arg use_tls: A boolean to specify whether the client should use a
+ a TLS connection. Default is True. Please, keep it.
+ """
+ self.use_tls = use_tls
+ self.hostname = hostname
+ self._username = username
+ self._password = password
+
+ self.token_file = Path.home() / DEFAULT_TOKEN_FILENAME
+
+ display.v("Connecting to {hostname} as user {user}".format(
+ hostname=to_native(self.hostname), user=to_native(self._username)))
+ try:
+ self.token = self._get_token_from_file()
+ except AnsibleFileNotFound:
+ display.vv("Force renew the token")
+ self._force_renew_token()
+
+ def _get_token_from_file(self):
+ display.vv("Trying to fetch token from {}".format(self.token_file))
+
+ # Check if the token file exists
+ if not self.token_file.is_file():
+ display.vv("Unable to access file {}".format(self.token_file))
+ raise AnsibleFileNotFound(file_name=self.token_file)
+
+ try:
+ with self.token_file.open() as f:
+ data = json.load(f)
+ except Exception as e:
+ display.vv("File {} not readable".format(self.token_file))
+ display.vvv("Original error was {}".format(to_native(e)))
+ raise AnsibleFileNotFound(file_name=self.token_file.as_posix() +
+ ' (Not readable)')
+
+ try:
+ token_data = data[self.hostname][self._username]
+ ret = {
+ 'token': token_data['token'],
+ 'expiration': self._parse_date(token_data["expiration"]),
+ }
+
+ except KeyError:
+ raise AnsibleLookupError("""Token for {user}@{host} not found
+ in token file ({token})""".format(user=self._username,
+ host=self.hostname,
+ token=self.token_file,
+ )
+ )
+ else:
+ display.vv("""Token successfully retreived from
+ file {token}""".format(token=self.token_file))
+ return ret
+
+ def _force_renew_token(self):
+ self.token = self._get_token_from_server()
+ self._save_token_to_file()
+
+ def _get_token_from_server(self):
+ display.vv("Requesting a new token for {user}@{host}".format(
+ user=self._username,
+ host=self.hostname,
+ ))
+ # Authentication request
+ response = requests.post(
+ self.get_url_for('token-auth'),
+ data={'username': self._username, 'password': self._password},
+ )
+ display.vv("Response code: {}".format(response.status_code))
+ if response.status_code == requests.codes.bad_request:
+ display.vv("Please provide valid credentials")
+ raise AnsibleLookupError("Unable to connect to the API for {host}"
+ .format(host=self.hostname))
+ try:
+ response.raise_for_status()
+ except Exception as e:
+ raise AnsibleError("""An error occured while trying to contact
+ the API. This was the original exception: {}"""
+ .format(to_native(e)))
+
+ response = response.json()
+ ret = {
+ 'token': response['token'],
+ 'expiration': self._parse_date(response['expiration']),
+ }
+
+ display.vv("Token successfully retreived for {user}@{host}".format(
+ user=self._username,
+ host=self.hostname,
+ )
+ )
+ return ret
+
+ def _parse_date(self, date, date_format="%Y-%m-%dT%H:%M:%S"):
+ return datetime.datetime.strptime(date.split('.')[0], date_format)
+
+ def _save_token_to_file(self):
+ display.vv("Saving token to file {}".format(self.token_file))
+ try:
+ # Read previous data to avoid erasures
+ with self.token_file.open() as f:
+ data = json.load(f)
+ except Exception:
+ display.v("""Beware, token file {} was not a valid JSON readable
+ file. Considered empty.""".format(self.token_file))
+ data = {}
+
+ if self.hostname not in data.keys():
+ data[self.hostname] = {}
+ data[self.hostname][self._username] = {
+ 'token': self.token['token'],
+ 'expiration': self.token['expiration'].isoformat(),
+ }
+
+ try:
+ with self.token_file.open('w') as f:
+ json.dump(data, f)
+ self.token_file.chmod(stat.S_IWRITE | stat.S_IREAD)
+ except Exception as e:
+ display.vv("Token file {} could not be written. Passing."
+ .format(self.token_file))
+ display.vvv("Original error was {}".format(to_native(e)))
+ else:
+ display.vv("Token successfully written to file {}"
+ .format(self.token_file))
+
+ def get_token(self):
+ """
+ Retrieves the token to use for the current connection.
+ Automatically renewed if needed.
+ """
+ if self.need_renew_token:
+ self._force_renew_token()
+
+ return self.token['token']
+
+ @property
+ def need_renew_token(self):
+ return self.token['expiration'] < \
+ datetime.datetime.now() + \
+ datetime.timedelta(seconds=TIME_FOR_RENEW)
+
+ def _request(self, method, url, headers={}, params={}, *args, **kwargs):
+ display.vv("Building the {method} request to {url}.".format(
+ method=method.upper(),
+ url=url,
+ ))
-import sys
-sys.path.append('./lookup_plugins/')
+ # Force the 'Authorization' field with the right token.
+ display.vvv("Forcing authentication token.")
+ headers.update({
+ 'Authorization': 'Token {}'.format(self.get_token())
+ })
-from re2oapi import Re2oAPIClient
+ # Use a json format unless the user already specified something
+ if 'format' not in params.keys():
+ display.vvv("Forcing JSON format response.")
+ params.update({'format': 'json'})
+
+ # Perform the request
+ display.v("{} {}".format(method.upper(), url))
+ response = getattr(requests, method)(
+ url, headers=headers, params=params, *args, **kwargs
+ )
+ display.vvv("Response code: {}".format(response.status_code))
+
+ if response.status_code == requests.codes.unauthorized:
+ # Force re-login to the server (case of a wrong token but valid
+ # credentials) and then retry the request without catching errors.
+ display.vv("Token refused. Trying to refresh the token.")
+ self._force_renew_token()
+
+ headers.update({
+ 'Authorization': 'Token {}'.format(self.get_token())
+ })
+ display.vv("Re-performing the request {method} {url}".format(
+ method=method.upper(),
+ url=url,
+ ))
+ response = getattr(requests, method)(
+ url, headers=headers, params=params, *args, **kwargs
+ )
+ display.vvv("Response code: ".format(response.status_code))
+
+ if response.status_code == requests.codes.forbidden:
+ err = "The {method} request to {url} was denied for {user}".format(
+ method=method.upper(),
+ url=url,
+ user=self._username
+ )
+ display.vvv(err)
+ raise AnsibleLookupError(to_native(err))
+
+ try:
+ response.raise_for_status()
+ except Exception as e:
+ raise AnsibleError("""An error occured while trying to contact
+ the API. This was the original exception: {}"""
+ .format(to_native(e)))
+
+ ret = response.json()
+ display.vvv("{method} request to {url} successful.".format(
+ method=method.upper(),
+ url=url
+ ))
+ return ret
+
+ def get_url_for(self, endpoint):
+ """
+ Retrieves the complete URL to use for a given endpoint's name.
+ """
+ return '{proto}://{host}/{namespace}/{endpoint}'.format(
+ proto=('https' if self.use_tls else 'http'),
+ host=self.hostname,
+ namespace='api',
+ endpoint=endpoint
+ )
+
+ def get(self, *args, **kwargs):
+ """
+ Perform a GET request to the API
+ """
+ return self._request('get', *args, **kwargs)
+
+ def list(self, endpoint, max_results=None, params={}):
+ """List all objects on the server that corresponds to the given
+ endpoint. The endpoint must be valid for listing objects.
+
+ :arg endpoint: The path of the endpoint.
+ :kwarg max_results: A limit on the number of result to return
+ :kwarg params: See `requests.get` params.
+ :returns: The list of all the objects as returned by the API.
+ """
+ display.v("Starting listing objects under '{}'"
+ .format(endpoint))
+ display.vvv("max_results = {}".format(max_results))
+
+ # For optimization, list all results in one page unless the user
+ # is forcing a different `page_size`.
+ if 'page_size' not in params.keys():
+ display.vvv("Forcing 'page_size' parameter to 'all'.")
+ params['page_size'] = max_results or 'all'
+
+ # Performs the request for the first page
+ response = self.get(
+ self.get_url_for(endpoint),
+ params=params,
+ )
+
+ results = response['results']
+
+ # Get all next pages and append the results
+ while response['next'] is not None and \
+ (max_results is None or len(results) < max_results):
+ response = self.get(response['next'])
+ results += response['results']
+
+ # Returns the exact number of results if applicable
+ ret = results[:max_results] if max_results else results
+ display.vvv("Listing objects under '{}' successful"
+ .format(endpoint))
+ return ret
+
+ def count(self, endpoint, params={}):
+ """Counts all objects on the server that corresponds to the given
+ endpoint. The endpoint must be valid for listing objects.
+
+ :arg endpoint: The path of the endpoint.
+ :kwarg params: See `requests.get` params.
+ :returns: Number of objects on the server as returned by the API.
+ """
+ display.v("Starting counting objects under '{}'"
+ .format(endpoint))
+
+ # For optimization, ask for only 1 result (so the server will take
+ # less time to process the request) unless the user is forcing
+ # a different `page_size`.
+ if 'page_size' not in params.keys():
+ display.vvv("Forcing 'page_size' parameter to '1'.")
+ params['page_size'] = 1
+
+ # Performs the request and return the `count` value in the response.
+ ret = self.get(
+ self.get_url_for(endpoint),
+ params=params,
+ )['count']
+
+ display.vvv("Counting objects under '{}' successful"
+ .format(endpoint))
+ return ret
+
+ def view(self, endpoint, params={}):
+ """Retrieves the details of an object from the server that corresponds
+ to the given endpoint.
+
+ :args endpoint: The path of the endpoint.
+ :kwargs params: See `requests.get` params.
+ :returns: The object serialized as returned by the API.
+ """
+ display.v("Starting viewing an object under '{}'"
+ .format(endpoint))
+ ret = self.get(
+ self.get_url_for(endpoint),
+ params=params
+ )
+
+ display.vvv("Viewing object under '{}' successful"
+ .format(endpoint))
+ return ret
class LookupModule(LookupBase):
"""
- If terms = dnszones then this module queries the re2o api and returns the list of all dns zones.
+ Available terms =
+ - dnszones: Queries the re2o API and returns the list of all dns zones
+ nicely formatted to be rendered in a template.
+ If a term is not in the previous list, make a raw query to the API
+ with endpoint term.
Usage:
- The following play will use the debug module to output all the zone names managed by crans.
+ The following play will use the debug module to output
+ all the zone names managed by Crans.
- hosts: sputnik.adm.crans.org
vars:
@@ -37,7 +371,6 @@ class LookupModule(LookupBase):
- debug: var=dnszones
"""
-
def run(self, terms, variables=None, api_hostname=None, api_username=None,
api_password=None, use_tls=True):
@@ -48,32 +381,47 @@ class LookupModule(LookupBase):
:kwarg api_hostname: The hostname of re2o instance.
:kwarg api_username: The username to connect to the API.
:kwarg api_password: The password to use to connect to the API.
- :kwarg use_tls: A boolean to specify whether to use tls or not. You should !
+ :kwarg use_tls: A boolean to specify whether to use tls. You should!
:returns: A list of results to the specific queries.
"""
if api_hostname is None:
- raise AnsibleError('You must specify a hostname to contact re2oAPI')
+ raise AnsibleError(to_native(
+ 'You must specify a hostname to contact re2oAPI'
+ ))
if api_username is None and api_password is None:
api_username = variables.get('vault_re2o_service_user')
api_password = variables.get('vault_re2o_service_password')
if api_username is None:
- raise AnsibleError('You must specify a valid username to connect to re2oAPI')
+ raise AnsibleError(to_native(
+ 'You must specify a valid username to connect to re2oAPI'
+ ))
if api_password is None:
- raise AnsibleError('You must specify a valid password to connect to re2oAPI')
+ raise AnsibleError(to_native(
+ 'You must specify a valid password to connect to re2oAPI'
+ ))
- api_client = Re2oAPIClient(api_hostname, api_username, api_password, use_tls=True)
+ api_client = Client(api_hostname, api_username,
+ api_password, use_tls=True)
res = []
for term in terms:
+ display.v("\nLookup for {} \n".format(term))
if term == 'dnszones':
res.append(self._getzones(api_client))
+ else:
+ res.append(self._rawquery(api_client, term))
return res
def _getzones(self, api_client):
+ display.v("Getting dns zone names")
zones = api_client.list('dns/zones')
zones_name = [zone["name"][1:] for zone in zones]
return zones_name
+
+ def _rawquery(self, api_client, endpoint):
+ display.v("Make a raw query to endpoint {}".format(endpoint))
+ return api_client.list(endpoint)
--
GitLab