From e3f5293ebc0878d49fbbc4a0f2d4ab8a6b58b606 Mon Sep 17 00:00:00 2001 From: Kumi Date: Sat, 26 Oct 2024 09:10:49 +0200 Subject: [PATCH] feat: automate Overpass API setup and monitoring Introduce a complete setup for the Overpass API on Debian. Add necessary scripts and configuration files to automate the installation and configuration process, including setting up Apache and creating systemd services for automatic startup. - Add an example Apache configuration file. - Implement a Debian installation script for the Overpass API. - Create a systemd service and timer for monitoring Overpass OSC and restart if no output is detected. - Set up additional systemd services for Overpass processes to ensure continuous operations. This setup simplifies deployment and enhances reliability by monitoring services and ensuring they restart automatically if needed. --- apache/example.conf | 24 ++++++++++++++++ install-debian.sh | 53 +++++++++++++++++++++++++++++++++++ oscmonitor/oscmonitor | 8 ++++++ systemd/oscmonitor.service | 6 ++++ systemd/oscmonitor.timer | 9 ++++++ systemd/overpass.service | 11 ++++++++ systemd/overpassareas.service | 15 ++++++++++ systemd/overpassloop.service | 12 ++++++++ systemd/overpassosc.service | 12 ++++++++ 9 files changed, 150 insertions(+) create mode 100644 apache/example.conf create mode 100644 install-debian.sh create mode 100644 oscmonitor/oscmonitor create mode 100644 systemd/oscmonitor.service create mode 100644 systemd/oscmonitor.timer create mode 100644 systemd/overpass.service create mode 100644 systemd/overpassareas.service create mode 100644 systemd/overpassloop.service create mode 100644 systemd/overpassosc.service diff --git a/apache/example.conf b/apache/example.conf new file mode 100644 index 0000000..8c192ae --- /dev/null +++ b/apache/example.conf @@ -0,0 +1,24 @@ + + ServerAdmin webmaster@localhost + DocumentRoot /opt/osm/html/ + + ServerName your.hostname.here + Header set Overpass-Worker "worker-name-here" + + ScriptAlias /api/ /opt/osm/cgi-bin/ + TimeOut 86400 + + + Require all granted + + + + AllowOverride None + Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch + Require all granted + + + ErrorLog /var/log/apache2/error.log + LogLevel warn + + diff --git a/install-debian.sh b/install-debian.sh new file mode 100644 index 0000000..02f2c3b --- /dev/null +++ b/install-debian.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Install Overpass API + +# Step 1: Install required software packages +echo "Installing required packages..." +apt install wget g++ make expat libexpat1-dev zlib1g-dev liblz4-dev -y + +# Step 2: Download the latest release of Overpass API +cd /opt +echo "Downloading Overpass API..." +wget https://dev.overpass-api.de/releases/osm-3s_latest.tar.gz + +# Extract the package +echo "Extracting package..." +tar -xzf osm-3s_latest.tar.gz + +# Navigate into the extracted directory +EXTRACTED_DIR=$(tar -tf osm-3s_latest.tar.gz | head -1 | cut -f1 -d"/") +ln -s "$EXTRACTED_DIR" osm +cd "$EXTRACTED_DIR" + +# Compile the software +echo "Compiling Overpass API..." +./configure --enable-lz4 +make +chmod 755 bin/*.sh cgi-bin/* + +# Step 3: Load Data +DB_DIR="db" + +echo "Cloning the database..." +mkdir -p $DB_DIR +bin/download_clone.sh --db-dir="$DB_DIR/" --meta=attic --source="https://dev.overpass-api.de/api_drolbr/" + +# Step 4: Create system user +echo "Creating system user..." +useradd -m osm +chown -R osm:osm . + +# Step 5: Set up systemd service files +echo "Setting up services..." +SCRIPT_DIR=$(dirname "$0") +cp "$SCRIPT_DIR/systemd/*" /etc/systemd/system/ +install "$SCRIPT_DIR/oscmonitor/oscmonitor" /usr/local/bin/ +systemctl enable --now overpass +systemctl enable --now overpassareas +systemctl enable --now overpassloop +systemctl enable --now overpassosc +systemctl enable --now oscmonitor.timer + +echo "Overpass API installation completed." +echo "Please set up Apache based on the apache/example.conf file." \ No newline at end of file diff --git a/oscmonitor/oscmonitor b/oscmonitor/oscmonitor new file mode 100644 index 0000000..e6c6fb5 --- /dev/null +++ b/oscmonitor/oscmonitor @@ -0,0 +1,8 @@ +#!/bin/bash + +SERVICE_NAME="overpassosc" + +if journalctl -u $SERVICE_NAME --since "1 hour ago" | grep -q -- '-- No entries --'; then + echo "No output from $SERVICE_NAME in the last hour. Restarting..." + systemctl restart $SERVICE_NAME +fi diff --git a/systemd/oscmonitor.service b/systemd/oscmonitor.service new file mode 100644 index 0000000..0c5768e --- /dev/null +++ b/systemd/oscmonitor.service @@ -0,0 +1,6 @@ +[Unit] +Description=Overpass OSC Monitor + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/oscmonitor diff --git a/systemd/oscmonitor.timer b/systemd/oscmonitor.timer new file mode 100644 index 0000000..2f01e94 --- /dev/null +++ b/systemd/oscmonitor.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Overpass OSC Monitor Timer + +[Timer] +OnBootSec=3600s +OnUnitActiveSec=3600s + +[Install] +WantedBy=timers.target diff --git a/systemd/overpass.service b/systemd/overpass.service new file mode 100644 index 0000000..661b4f7 --- /dev/null +++ b/systemd/overpass.service @@ -0,0 +1,11 @@ +[Service] +User = osm +Group = osm +WorkingDirectory = /opt/osm/ +ExecStartPre=/bin/sh -c "rm -f /opt/osm/db/osm*_base || true" +ExecStartPre=/bin/sh -c "rm -f /dev/shm/osm*_base || true" +ExecStart=/opt/osm/bin/dispatcher --osm-base --attic --db-dir=/opt/osm/db --rate-limit=-1 --allow-duplicate-queries=yes +LimitNOFILE=10000 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemd/overpassareas.service b/systemd/overpassareas.service new file mode 100644 index 0000000..b951faf --- /dev/null +++ b/systemd/overpassareas.service @@ -0,0 +1,15 @@ +[Unit] +Description = Overpass Areas +After = overpass.service + +[Service] +User = osm +Group = nogroup +WorkingDirectory = /opt/osm/ +ExecStartPre=/bin/sh -c "rm -f /opt/osm/db/osm3s*_areas || true" +ExecStartPre=/bin/sh -c "rm -f /dev/shm/osm3s*_areas || true" +ExecStart=/opt/osm/bin/dispatcher --areas --db-dir=/opt/osm/db --rate-limit=-1 --allow-duplicate-queries=yes +LimitNOFILE=10000 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemd/overpassloop.service b/systemd/overpassloop.service new file mode 100644 index 0000000..b927cf3 --- /dev/null +++ b/systemd/overpassloop.service @@ -0,0 +1,12 @@ +[Unit] +Description = Overpass Area Loop +After = overpass.service + +[Service] +User = osm +Group = osm +WorkingDirectory = /opt/osm/ +ExecStart=/opt/osm/bin/rules_loop.sh "db/" + +[Install] +WantedBy=multi-user.target diff --git a/systemd/overpassosc.service b/systemd/overpassosc.service new file mode 100644 index 0000000..13a1ea2 --- /dev/null +++ b/systemd/overpassosc.service @@ -0,0 +1,12 @@ +[Unit] +Description = Overpass OSC +After = overpass.service + +[Service] +User = osm +Group = osm +WorkingDirectory = /opt/osm/ +ExecStart=/opt/osm/bin/fetch_osc_and_apply.sh "http://planet.openstreetmap.org/replication/minute" --meta=attic + +[Install] +WantedBy=multi-user.target \ No newline at end of file