From 3fa8c60c6d6edfc2778fec2ec1d9f99c85012ea8 Mon Sep 17 00:00:00 2001
From: shirenn <shirenn@crans.org>
Date: Tue, 1 Jun 2021 17:49:21 +0200
Subject: [PATCH] [mirror_backend]
---
group_vars/mirror_backend.yml | 90 +++++++++++++++++++
host_vars/charybde.adm.crans.org.yml | 72 ++++++---------
host_vars/tealc.adm.crans.org.yml | 16 ++++
hosts | 12 ++-
roles/apt-mirror/tasks/main.yml | 41 +++++++++
roles/apt-mirror/templates/apt/mirror.list.j2 | 22 +++++
.../apt-mirror/templates/cron.d/apt-mirror.j2 | 4 +
.../templates/update-motd.d/05-service.j2 | 3 +
roles/ftpsync/tasks/main.yml | 35 ++++++--
roles/ftpsync/templates/ftpsync-cron.j2 | 4 +-
roles/ftpsync/templates/ftpsync.conf.j2 | 11 ++-
roles/rsync-mirror/tasks/main.yml | 17 ++++
.../templates/rsync-mirror-cron.j2 | 4 +-
13 files changed, 267 insertions(+), 64 deletions(-)
create mode 100644 group_vars/mirror_backend.yml
create mode 100644 roles/apt-mirror/tasks/main.yml
create mode 100644 roles/apt-mirror/templates/apt/mirror.list.j2
create mode 100644 roles/apt-mirror/templates/cron.d/apt-mirror.j2
create mode 100755 roles/apt-mirror/templates/update-motd.d/05-service.j2
diff --git a/group_vars/mirror_backend.yml b/group_vars/mirror_backend.yml
new file mode 100644
index 00000000..d3898eae
--- /dev/null
+++ b/group_vars/mirror_backend.yml
@@ -0,0 +1,90 @@
+glob_ftpsync:
+ root: /mirror/pub
+ mirror:
+ name: CRANS
+ info:
+ maintainer: Les Nounous <contact@crans.org>
+ country: FR
+ location: Cachan, ÃŽle-de-France
+ targets:
+ - name: main
+ dest: debian
+ cron_time: "25 1,13"
+ rsync_host: syncproxy.eu.debian.org
+ rsync_path: debian
+ - name: security
+ dest: debian-security
+ cron_time: "40 *"
+ rsync_host: rsync.security.debian.org
+ rsync_path: debian-security
+ - name: backports
+ dest: debian-backports
+ cron_time: " 7 3,15"
+ rsync_host: syncproxy.eu.debian.org
+ rsync_path: debian-backports
+
+glob_rsync_mirror:
+ root: /mirror/pub
+ targets:
+ - name: videolan
+ dest: videolan
+ cron_time: "03 10,14,18,22,2,6"
+ rsync_host: rsync.videolan.org
+ rsync_path: videolan-ftp
+ - name: debian
+ dest: distributions/linux/debian
+ cron_time: "00 5"
+ rsync_host: cdimage.debian.org
+ rsync_path: cdimage/release
+ - name: debian-cloud
+ dest: distributions/linux/debian/cloud
+ cron_time: "00 5"
+ rsync_host: cdimage.debian.org
+ rsync_path: cdimage/cloud/Openstack
+ exclude:
+ - archive
+ - name: ubuntu
+ dest: distributions/linux/ubuntu
+ cron_time: "00 5"
+ rsync_host: cdimage.ubuntu.com
+ rsync_path: cdimage/releases
+ - name: xubuntu
+ dest: distributions/linux/xubuntu
+ cron_time: "00 5"
+ rsync_host: cdimage.ubuntu.com
+ rsync_path: cdimage/xubuntu/releases
+ - name: kubuntu
+ dest: distributions/linux/kubuntu
+ cron_time: "00 5"
+ rsync_host: cdimage.ubuntu.com
+ rsync_path: cdimage/kubuntu/releases
+ - name: lubuntu
+ dest: distributions/linux/lubuntu
+ cron_time: "00 5"
+ rsync_host: cdimage.ubuntu.com
+ rsync_path: cdimage/lubuntu/releases
+ - name: ubuntu-mate
+ dest: distributions/linux/ubuntu-mate
+ cron_time: "00 5"
+ rsync_host: cdimage.ubuntu.com
+ rsync_path: cdimage/ubuntu-mate/releases
+
+glob_apt_mirror:
+ root: /mirror/pub
+ targets:
+ - name: grafana
+ symlink: ""
+ scheme: https
+ host: packages.grafana.com
+ path: oss/deb
+ suite: stable
+ components:
+ - main
+ - name: proxmox
+ symlink: ""
+ scheme: http
+ host: download.proxmox.com
+ path: debian/pve
+ suite: buster
+ components:
+ - pve-no-subscription
diff --git a/host_vars/charybde.adm.crans.org.yml b/host_vars/charybde.adm.crans.org.yml
index 00db4ce7..c99eea37 100644
--- a/host_vars/charybde.adm.crans.org.yml
+++ b/host_vars/charybde.adm.crans.org.yml
@@ -1,63 +1,47 @@
---
+loc_ftpsync:
+ root: /pool/mirror/pub
-to_backup:
- - {
- name: "var",
- path: "/var",
- auth_users: "backupcrans",
- secrets_file: "/etc/rsyncd.secrets",
- hosts_allow: ["zephir.adm.crans.org", "10.231.136.6"],
- }
- - {
- name: "slash",
- path: "/",
- auth_users: "backupcrans",
- secrets_file: "/etc/rsyncd.secrets",
- hosts_allow: ["zephir.adm.crans.org", "10.231.136.6"],
- }
- - {
- name: "ftp",
- path: "/pubftp",
- comment: "CRANS FTP",
- uid: "nobody",
- gid: "nogroup",
- hosts_allow: "*",
- read_only: "yes"
- }
- - {
- name: "videolan",
- path: "/pubftp/videolan",
- comment: "VideoLAN repository",
- uid: "nobody",
- gid: "nogroup",
- hosts_allow: "*",
- read_only: "yes"
- }
+loc_rsync_mirror:
+ root: /pool/mirror/pub
+
+loc_apt_mirror:
+ root: /pool/mirror/pub
loc_nginx:
service_name: ftp
ssl: []
servers:
- server_name:
- - "ftp"
- - "ftp.*"
- "mirror"
- "mirror.*"
- - "archive.ubuntu.com"
- - "fr.archive.ubuntu.com"
- - "security.ubuntu.com"
+ root: "/pool/mirror/pub"
+ locations:
+ - filter: "/"
+ params:
+ - "autoindex on"
+ - "autoindex_exact_size off"
+ - "add_before_body /.html/HEADER.html"
+ - "add_after_body /.html/FOOTER.html"
+ - server_name:
+ - "ftp"
+ - "ftp.*"
- "ftps"
- "ftps.*"
- root: "/pubftp"
+ root: /pool/memorial
locations:
- filter: "/"
params:
- - "autoindex on"
- - "autoindex_exact_size off"
- - "add_before_body /.html/HEADER.html"
- - "add_after_body /.html/FOOTER.html"
- - filter: "/pub/events/"
+ - "autoindex on"
+ - "autoindex_exact_size off"
+ - "add_before_body /.html/HEADER.html"
+ - "add_after_body /.html/FOOTER.html"
+ - filter: ~ ^(\/pub)?(\/debian|\/ubuntu|\/videolan)(.*)$
+ params:
+ - return 301 http://mirror.crans.org$2$3
+ - filter: "/events"
params:
- "mp4"
- "mp4_buffer_size 1m"
- "mp4_max_buffer_size 5m"
+
diff --git a/host_vars/tealc.adm.crans.org.yml b/host_vars/tealc.adm.crans.org.yml
index 1f116850..dce9520d 100644
--- a/host_vars/tealc.adm.crans.org.yml
+++ b/host_vars/tealc.adm.crans.org.yml
@@ -62,3 +62,19 @@ loc_mtail:
- dhcpd.mtail
remove:
- radiusd.mtail
+
+loc_nginx:
+ service_name: ftp
+ ssl: []
+ servers:
+ - server_name:
+ - "mirror2"
+ - "mirror2.*"
+ root: "/pool/mirror/pub"
+ locations:
+ - filter: "/"
+ params:
+ - "autoindex on"
+ - "autoindex_exact_size off"
+ - "add_before_body /.html/HEADER.html"
+ - "add_after_body /.html/FOOTER.html"
diff --git a/hosts b/hosts
index d1946d3c..716747aa 100644
--- a/hosts
+++ b/hosts
@@ -31,6 +31,7 @@ belenios.adm.crans.org
routeur-gulp.cachan-adm.crans.org
[certbot]
+charybde.adm.crans.org
sputnik.adm.crans.org
[certbot:children]
@@ -113,12 +114,20 @@ tealc.adm.crans.org
monitoring.adm.crans.org
fyre.cachan-adm.crans.org
+[mirror_backend]
+charybde.adm.crans.org
+eclat.adm.crans.org
+
+[mirror_frontend]
+charybde.adm.crans.org
+tealc.adm.crans.org
+
[nginx:children]
django_cas
galene
jitsi
mailman
-ntp_server
+mirror_frontend
re2o_front
reverseproxy
roundcube
@@ -127,6 +136,7 @@ wiki
[ntp_server]
charybde.adm.crans.org
+eclat.adm.crans.org
terenez.cachan-adm.crans.org
[opendkim:children]
diff --git a/roles/apt-mirror/tasks/main.yml b/roles/apt-mirror/tasks/main.yml
new file mode 100644
index 00000000..47f7d65a
--- /dev/null
+++ b/roles/apt-mirror/tasks/main.yml
@@ -0,0 +1,41 @@
+---
+- name: Install apt-mirror
+ apt:
+ update_cache: true
+ name: apt-mirror
+ register: apt_result
+ retries: 3
+ until: apt_result is succeeded
+
+- name: Create mirrors directory
+ file:
+ path: "{{ apt_mirror.root }}/{{ item.host }}"
+ owner: apt-mirror
+ group: mirror
+ mode: 0755
+ state: directory
+ loop: "{{ apt_mirror.targets }}"
+
+- name: Create mirror symlink
+ file:
+ src: "{{ apt_mirror.root }}/{{ item.host }}/{{ item.symlink }}"
+ dest: "{{ apt_mirror.root }}/{{ item.name }}"
+ mode: 0755
+ state: link
+ loop: "{{ apt_mirror.targets }}"
+
+- name: Copy apt-mirror configurations
+ template:
+ src: apt/mirror.list.j2
+ dest: "/etc/apt/mirror.list"
+
+- name: Configure apt-mirror cron
+ template:
+ src: cron.d/apt-mirror.j2
+ dest: /etc/cron.d/apt-mirror
+
+- name: Indicate role in motd
+ template:
+ src: update-motd.d/05-service.j2
+ dest: /etc/update-motd.d/05-apt-mirror
+ mode: 0755
diff --git a/roles/apt-mirror/templates/apt/mirror.list.j2 b/roles/apt-mirror/templates/apt/mirror.list.j2
new file mode 100644
index 00000000..ace1136f
--- /dev/null
+++ b/roles/apt-mirror/templates/apt/mirror.list.j2
@@ -0,0 +1,22 @@
+############# config ##################
+#
+# set base_path /var/spool/apt-mirror
+#
+set mirror_path {{ apt_mirror.root }}
+# set skel_path $base_path/skel
+# set var_path $base_path/var
+# set cleanscript $var_path/clean.sh
+# set defaultarch <running host architecture>
+# set postmirror_script $var_path/postmirror.sh
+# set run_postmirror 0
+set nthreads 20
+set _tilde 0
+#
+############# end config ##############
+
+{% for target in apt_mirror.targets %}
+deb {{ target.scheme }}://{{ target.host }}/{{ target.path }} {{ target.suite }}{% for component in target.components %} {{ component }}{% endfor %}
+
+clean {{ target.scheme }}://{{ target.host }}/{{ target.path }}
+
+{% endfor %}
diff --git a/roles/apt-mirror/templates/cron.d/apt-mirror.j2 b/roles/apt-mirror/templates/cron.d/apt-mirror.j2
new file mode 100644
index 00000000..b772acbe
--- /dev/null
+++ b/roles/apt-mirror/templates/cron.d/apt-mirror.j2
@@ -0,0 +1,4 @@
+#
+# Regular cron jobs for the apt-mirror package
+#
+0 4 * * * apt-mirror /usr/bin/apt-mirror > /var/spool/apt-mirror/var/cron.log
diff --git a/roles/apt-mirror/templates/update-motd.d/05-service.j2 b/roles/apt-mirror/templates/update-motd.d/05-service.j2
new file mode 100755
index 00000000..ff3222f8
--- /dev/null
+++ b/roles/apt-mirror/templates/update-motd.d/05-service.j2
@@ -0,0 +1,3 @@
+#!/usr/bin/tail +14
+{{ ansible_header | comment }}
+[0m> [38;5;82apt-mirror[0m a été déployé sur cette machine. Voir [38;5;6m/etc/apt/mirror.list[0m.
diff --git a/roles/ftpsync/tasks/main.yml b/roles/ftpsync/tasks/main.yml
index 2e6ca8b9..c6e4e1b4 100644
--- a/roles/ftpsync/tasks/main.yml
+++ b/roles/ftpsync/tasks/main.yml
@@ -16,11 +16,31 @@
home: /var/mirror # unused, should be something empty
shell: /bin/false
+- name: Create /etc/ftpsync directory
+ file:
+ path: "{{ item }}"
+ owner: root
+ group: root
+ mode: 0755
+ state: directory
+ loop:
+ - "/etc/ftpsync"
+ - "{{ ftpsync.root }}/.html"
+
+- name: Create root directory
+ file:
+ path: "{{ ftpsync.root }}"
+ owner: mirror
+ group: mirror
+ mode: 0755
+ state: directory
+
+
- name: Copy ftpsync configurations
template:
src: ftpsync.conf.j2
dest: "/etc/ftpsync/ftpsync-{{ item.name }}.conf"
- loop: "{{ ftpsync }}"
+ loop: "{{ ftpsync.targets }}"
- name: Configure ftpsync cron
template:
@@ -35,13 +55,10 @@
- name: Copy configuration files
template:
- src: "{{ item.src }}"
- dest: "{{ item.dest }}"
+ src: "html/{{ item }}.j2"
+ dest: "{{ ftpsync.root }}/.html/{{ item }}"
mode: 0644
loop:
- - src: html/HEADER.html.j2
- dest: /pubftp/.html/HEADER.html
- - src: html/FOOTER.html.j2
- dest: /pubftp/.html/FOOTER.html
- - src: html/style.min.css.j2
- dest: /pubftp/.html/style.min.css
+ - HEADER.html
+ - FOOTER.html
+ - style.min.css
diff --git a/roles/ftpsync/templates/ftpsync-cron.j2 b/roles/ftpsync/templates/ftpsync-cron.j2
index d4639dc0..773c3f92 100644
--- a/roles/ftpsync/templates/ftpsync-cron.j2
+++ b/roles/ftpsync/templates/ftpsync-cron.j2
@@ -1,4 +1,4 @@
{{ ansible_header | comment }}
-{% for i in ftpsync %}
-{{ i.cron_time }} * * * mirror ftpsync sync:archive:{{ i.name }}
+{% for target in ftpsync.targets %}
+{{ target.cron_time }} * * * mirror ftpsync sync:archive:{{ target.name }}
{% endfor %}
diff --git a/roles/ftpsync/templates/ftpsync.conf.j2 b/roles/ftpsync/templates/ftpsync.conf.j2
index f2045da5..2589c4f2 100644
--- a/roles/ftpsync/templates/ftpsync.conf.j2
+++ b/roles/ftpsync/templates/ftpsync.conf.j2
@@ -1,7 +1,7 @@
{{ ansible_header | comment }}
-MIRRORNAME="CRANS"
-TO="{{ item.dest }}"
+MIRRORNAME="{{ ftpsync.mirror.name }}"
+TO="{{ ftpsync.root}}/{{ item.dest }}/"
MAILTO="root"
# HUB=false
@@ -18,10 +18,9 @@ RSYNC_PATH="{{ item.rsync_path }}"
## Mirror information options
########################################################################
-INFO_MAINTAINER="Les Nounous <contact@crans.org>"
-# INFO_SPONSOR="Example <https://example.com>"
-INFO_COUNTRY=FR
-INFO_LOCATION="Cachan, ÃŽle-de-France"
+INFO_MAINTAINER="{{ ftpsync.mirror.info.maintainer }}"
+INFO_COUNTRY="{{ ftpsync.mirror.info.country }}"
+INFO_LOCATION="{{ ftpsync.mirror.info.location }}"
INFO_THROUGHPUT=1Gb
########################################################################
diff --git a/roles/rsync-mirror/tasks/main.yml b/roles/rsync-mirror/tasks/main.yml
index 52be85b7..45a1f57d 100644
--- a/roles/rsync-mirror/tasks/main.yml
+++ b/roles/rsync-mirror/tasks/main.yml
@@ -6,6 +6,23 @@
home: /var/mirror # unused, should be something empty
shell: /bin/false
+- name: Create root directory
+ file:
+ path: "{{ ftpsync.root }}"
+ owner: mirror
+ group: mirror
+ mode: 0755
+ state: directory
+
+- name: Create target directory
+ file:
+ path: "{{ rsync_mirror.root }}/{{ item.dest }}"
+ owner: mirror
+ group: mirror
+ mode: 0755
+ state: directory
+ loop: "{{ rsync_mirror.targets }}"
+
- name: Configure rsync cron
template:
src: rsync-mirror-cron.j2
diff --git a/roles/rsync-mirror/templates/rsync-mirror-cron.j2 b/roles/rsync-mirror/templates/rsync-mirror-cron.j2
index cb98968f..35ea881a 100644
--- a/roles/rsync-mirror/templates/rsync-mirror-cron.j2
+++ b/roles/rsync-mirror/templates/rsync-mirror-cron.j2
@@ -1,4 +1,4 @@
{{ ansible_header | comment }}
-{% for i in rsync_mirror %}
-{{ i.cron_time }} * * * mirror rsync --times --links --hard-links --partial --block-size=8192 --recursive --exclude *-dvd-* --exclude source/ --verbose --stats --delete-after rsync://{{ i.rsync_host }}/{{ i.rsync_path }} {{ i.dest }} | tee -a "/var/log/mirror/{{ i.name }}.log" > /dev/null
+{% for i in rsync_mirror.targets %}
+{{ i.cron_time }} * * * mirror rsync --times --links --hard-links --partial --block-size=8192 --recursive --exclude *-dvd-* --exclude source/ --verbose --stats --delete-after {%for exclude in i.exclude | default([]) %}--exclude {{ exclude }} {% endfor %}rsync://{{ i.rsync_host }}/{{ i.rsync_path }} {{ rsync_mirror.root }}/{{ i.dest }} | tee -a "/var/log/mirror/{{ i.name }}.log" > /dev/null
{% endfor %}
--
GitLab