From ebff7f5d72ea7ac11f3c1e67f5cb283578c167f5 Mon Sep 17 00:00:00 2001 From: Paul Walko Date: Sun, 8 Sep 2019 14:20:20 -0400 Subject: [PATCH] reogranize docker stuff + add backup script --- .gitignore | 21 ++++--- .../docker}/syncthing/launch.sh | 0 fogcutter/backup.sh | 20 ++++++ .../docker}/airsonic/launch.sh | 0 .../docker}/minecraft/launch.sh | 0 fogcutter/docker/monitoring/launch.sh | 61 +++++++++++++++++++ .../docker/monitoring/prometheus/alert.rules | 0 .../monitoring/prometheus/prometheus.yml | 24 ++++++++ .../docker}/nginx/README.md | 0 .../docker}/nginx/launch.sh | 0 .../docker}/nginx/nginx.conf | 0 .../site-confs/airsonic.seaturtle.pw.enabled | 0 .../docker}/nginx/site-confs/default | 0 .../site-confs/plex.seaturtle.pw.enabled | 0 .../nginx/site-confs/seaturtle.pw.enabled | 0 .../docker}/plex/launch.sh | 0 .../docker}/syncthing/launch.sh | 0 .../docker}/ut04/launch.sh | 0 ethernet_dhcp => misc/ethernet_dhcp | 0 ethernet_usb_dhcp => misc/ethernet_usb_dhcp | 0 fb.py => misc/fb.py | 0 install.sh => misc/install.sh | 0 loadCSSJS.js => misc/loadCSSJS.js | 0 space_scraper.py => misc/space_scraper.py | 0 .../ubertooth_install.sh | 0 25 files changed, 117 insertions(+), 9 deletions(-) rename {docker-cabinet => cabinet/docker}/syncthing/launch.sh (100%) create mode 100755 fogcutter/backup.sh rename {docker-fogcutter => fogcutter/docker}/airsonic/launch.sh (100%) rename {docker-fogcutter => fogcutter/docker}/minecraft/launch.sh (100%) create mode 100755 fogcutter/docker/monitoring/launch.sh create mode 100644 fogcutter/docker/monitoring/prometheus/alert.rules create mode 100644 fogcutter/docker/monitoring/prometheus/prometheus.yml rename {docker-fogcutter => fogcutter/docker}/nginx/README.md (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/launch.sh (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/nginx.conf (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/site-confs/airsonic.seaturtle.pw.enabled (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/site-confs/default (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/site-confs/plex.seaturtle.pw.enabled (100%) rename {docker-fogcutter => fogcutter/docker}/nginx/site-confs/seaturtle.pw.enabled (100%) rename {docker-fogcutter => fogcutter/docker}/plex/launch.sh (100%) rename {docker-fogcutter => fogcutter/docker}/syncthing/launch.sh (100%) rename {docker-fogcutter => fogcutter/docker}/ut04/launch.sh (100%) rename ethernet_dhcp => misc/ethernet_dhcp (100%) rename ethernet_usb_dhcp => misc/ethernet_usb_dhcp (100%) rename fb.py => misc/fb.py (100%) rename install.sh => misc/install.sh (100%) rename loadCSSJS.js => misc/loadCSSJS.js (100%) rename space_scraper.py => misc/space_scraper.py (100%) rename ubertooth_install.sh => misc/ubertooth_install.sh (100%) diff --git a/.gitignore b/.gitignore index ba95879..eb6d3ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ **/*htpasswd* -docker-fogcutter/airsonic/config -docker-fogcutter/minecraft/data -docker-fogcutter/nginx/config -docker-fogcutter/plex/config -docker-fogcutter/syncthing/config -docker-fogcutter/syncthing/sync -docker-madone/nginx/config -docker-madone/syncthing/config -docker-madone/syncthing/sync +**/backups +fogcutter/docker/airsonic/config +fogcutter/docker/minecraft/data +fogcutter/docker/nginx/config +fogcutter/docker/plex/config +fogcutter/docker/syncthing/config +fogcutter/docker/syncthing/sync +fogcutter/docker/monitoring/grafana/data +fogcutter/docker/monitoring/prometheus/data +madone/docker/nginx/config +madone/docker/syncthing/config +madone/docker/syncthing/sync diff --git a/docker-cabinet/syncthing/launch.sh b/cabinet/docker/syncthing/launch.sh similarity index 100% rename from docker-cabinet/syncthing/launch.sh rename to cabinet/docker/syncthing/launch.sh diff --git a/fogcutter/backup.sh b/fogcutter/backup.sh new file mode 100755 index 0000000..c2de605 --- /dev/null +++ b/fogcutter/backup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# GPG key must be imported + +DATE=$(date --iso-8601) +FINGERPRINT=1807F8D422B89582ADEC4A790B1A6ED3E577B121 +SERVICE_PATH=/home/paul/scripts/fogcutter/docker +BACKUP_PATH=/bigdata/backups +WORKDIR=/tmp + + +cd $WORKDIR + +cp -r "$SERVICE_PATH"/minecraft/data ./minecraft-data +tar cz ./minecraft-data | gpg -e -r $FINGERPRINT -o "$BACKUP_PATH"/minecraft-data-"$DATE".tgz.gpg +rm -rf ./minecraft-data + +cp -r "$SERVICE_PATH"/syncthing/sync ./syncthing-data +tar cz ./syncthing-data | gpg -e -r $FINGERPRINT -o "$BACKUP_PATH"/syncthing-data-"$DATE".tgz.gpg +rm -rf ./syncthing-data diff --git a/docker-fogcutter/airsonic/launch.sh b/fogcutter/docker/airsonic/launch.sh similarity index 100% rename from docker-fogcutter/airsonic/launch.sh rename to fogcutter/docker/airsonic/launch.sh diff --git a/docker-fogcutter/minecraft/launch.sh b/fogcutter/docker/minecraft/launch.sh similarity index 100% rename from docker-fogcutter/minecraft/launch.sh rename to fogcutter/docker/minecraft/launch.sh diff --git a/fogcutter/docker/monitoring/launch.sh b/fogcutter/docker/monitoring/launch.sh new file mode 100755 index 0000000..bfebf83 --- /dev/null +++ b/fogcutter/docker/monitoring/launch.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Assumes pew-net exists +# Assumes mounted dirs are created as non-root user +# Prometheus + Node Exporter + cAdvisor + Grafana + +docker network create pew-monitoring-net + +# grafana-cli plugins install grafana-piechart-panel +sudo chown 472:472 $PWD/grafana/{data,provisioning} \ +&& \ +docker run \ + --name monitoring_grafana \ + --detach \ + --restart unless-stopped \ + --user 472 \ + --volume $PWD/grafana/data:/var/lib/grafana:rw \ + --volume $PWD/grafana/provisioning:/etc/grafana/provisioning:rw \ + --publish 3000:3000 \ + --network pew-monitoring-net \ + grafana/grafana:6.3.5 \ +&& \ +docker network connect pew-net monitoring_grafana + +# --publish 9090:9090 \ +sudo chown nobody:nogroup $PWD/prometheus/data \ +&& \ +docker run \ + --name monitoring_prometheus \ + --detach \ + --restart unless-stopped \ + --volume $PWD/prometheus/data:/prometheus:rw \ + --volume $PWD/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro \ + --volume $PWD/prometheus/alert.rules:/etc/prometheus/alert.rules:ro \ + --network pew-monitoring-net \ + prom/prometheus:v2.12.0 --config.file=/etc/prometheus/prometheus.yml \ + --storage.tsdb.path=/prometheus + +docker run \ + --name monitoring_node-exporter \ + --detach \ + --restart unless-stopped \ + --cap-add SYS_TIME \ + --pid host \ + --volume /:/host:ro,rslave \ + --network pew-monitoring-net \ + quay.io/prometheus/node-exporter:v0.18.1 --path.rootfs=/host + +docker run \ + --name monitoring_cadvisor \ + --detach \ + --restart unless-stopped \ + --volume /:/rootfs:ro \ + --volume /var/run:/var/run:ro \ + --volume /sys:/sys:ro \ + --volume /var/lib/docker:/var/lib/docker:ro \ + --volume /dev/disk:/dev/disk:ro \ + --network pew-monitoring-net \ + google/cadvisor:v0.32.0 + +# alert manager diff --git a/fogcutter/docker/monitoring/prometheus/alert.rules b/fogcutter/docker/monitoring/prometheus/alert.rules new file mode 100644 index 0000000..e69de29 diff --git a/fogcutter/docker/monitoring/prometheus/prometheus.yml b/fogcutter/docker/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..e961858 --- /dev/null +++ b/fogcutter/docker/monitoring/prometheus/prometheus.yml @@ -0,0 +1,24 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: +- job_name: prometheus + static_configs: + - targets: ['monitoring_prometheus:9090'] + +- job_name: cadvisor + static_configs: + - targets: ['monitoring_cadvisor:8080'] + +- job_name: node-exporter + static_configs: + - targets: ['monitoring_node-exporter:9100'] + +#alerting: +# alertmanagers: +# - static_configs: +# - targets: ['alertmanager:9093'] +# +#rule_files: +#- 'alert.rules' diff --git a/docker-fogcutter/nginx/README.md b/fogcutter/docker/nginx/README.md similarity index 100% rename from docker-fogcutter/nginx/README.md rename to fogcutter/docker/nginx/README.md diff --git a/docker-fogcutter/nginx/launch.sh b/fogcutter/docker/nginx/launch.sh similarity index 100% rename from docker-fogcutter/nginx/launch.sh rename to fogcutter/docker/nginx/launch.sh diff --git a/docker-fogcutter/nginx/nginx.conf b/fogcutter/docker/nginx/nginx.conf similarity index 100% rename from docker-fogcutter/nginx/nginx.conf rename to fogcutter/docker/nginx/nginx.conf diff --git a/docker-fogcutter/nginx/site-confs/airsonic.seaturtle.pw.enabled b/fogcutter/docker/nginx/site-confs/airsonic.seaturtle.pw.enabled similarity index 100% rename from docker-fogcutter/nginx/site-confs/airsonic.seaturtle.pw.enabled rename to fogcutter/docker/nginx/site-confs/airsonic.seaturtle.pw.enabled diff --git a/docker-fogcutter/nginx/site-confs/default b/fogcutter/docker/nginx/site-confs/default similarity index 100% rename from docker-fogcutter/nginx/site-confs/default rename to fogcutter/docker/nginx/site-confs/default diff --git a/docker-fogcutter/nginx/site-confs/plex.seaturtle.pw.enabled b/fogcutter/docker/nginx/site-confs/plex.seaturtle.pw.enabled similarity index 100% rename from docker-fogcutter/nginx/site-confs/plex.seaturtle.pw.enabled rename to fogcutter/docker/nginx/site-confs/plex.seaturtle.pw.enabled diff --git a/docker-fogcutter/nginx/site-confs/seaturtle.pw.enabled b/fogcutter/docker/nginx/site-confs/seaturtle.pw.enabled similarity index 100% rename from docker-fogcutter/nginx/site-confs/seaturtle.pw.enabled rename to fogcutter/docker/nginx/site-confs/seaturtle.pw.enabled diff --git a/docker-fogcutter/plex/launch.sh b/fogcutter/docker/plex/launch.sh similarity index 100% rename from docker-fogcutter/plex/launch.sh rename to fogcutter/docker/plex/launch.sh diff --git a/docker-fogcutter/syncthing/launch.sh b/fogcutter/docker/syncthing/launch.sh similarity index 100% rename from docker-fogcutter/syncthing/launch.sh rename to fogcutter/docker/syncthing/launch.sh diff --git a/docker-fogcutter/ut04/launch.sh b/fogcutter/docker/ut04/launch.sh similarity index 100% rename from docker-fogcutter/ut04/launch.sh rename to fogcutter/docker/ut04/launch.sh diff --git a/ethernet_dhcp b/misc/ethernet_dhcp similarity index 100% rename from ethernet_dhcp rename to misc/ethernet_dhcp diff --git a/ethernet_usb_dhcp b/misc/ethernet_usb_dhcp similarity index 100% rename from ethernet_usb_dhcp rename to misc/ethernet_usb_dhcp diff --git a/fb.py b/misc/fb.py similarity index 100% rename from fb.py rename to misc/fb.py diff --git a/install.sh b/misc/install.sh similarity index 100% rename from install.sh rename to misc/install.sh diff --git a/loadCSSJS.js b/misc/loadCSSJS.js similarity index 100% rename from loadCSSJS.js rename to misc/loadCSSJS.js diff --git a/space_scraper.py b/misc/space_scraper.py similarity index 100% rename from space_scraper.py rename to misc/space_scraper.py diff --git a/ubertooth_install.sh b/misc/ubertooth_install.sh similarity index 100% rename from ubertooth_install.sh rename to misc/ubertooth_install.sh