Public Access
Jonathan Miller
7800eadbd7
- Guard upload-artifact@v4 / download-artifact@v4 with github.server_url == 'https://github.com' so they skip on Gitea - Add Gitea fallbacks (checkout or log message) where artifacts are used - Make enforce-tags step continue-on-error so sync doesn't fail on tag issues - Replace upload-artifact in bulk-repo-sync with step summary on Gitea - Fix escaped variable references in bulk-repo-sync.yml Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
346 lines
14 KiB
Plaintext
346 lines
14 KiB
Plaintext
# Copyright (C) 2026 Moko Consulting <hello@mokoconsulting.tech>
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
# FILE INFORMATION
|
|
# DEFGROUP: Gitea.Workflow
|
|
# INGROUP: MokoStandards-API.Deployment
|
|
# REPO: https://git.mokoconsulting.tech/MokoConsulting/MokoStandards-API
|
|
# PATH: /templates/workflows/shared/export-mysql.yml.template
|
|
# VERSION: 04.06.12
|
|
# BRIEF: Export MySQL database from dev/demo server and save as artifact or commit
|
|
|
|
name: Export MySQL Database
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
environment:
|
|
description: 'Which server to export from'
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- dev
|
|
- demo
|
|
default: 'dev'
|
|
database:
|
|
description: 'Database name (overrides variable)'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
save_to_repo:
|
|
description: 'Commit SQL dump to repo (otherwise artifact only)'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
branch:
|
|
description: 'Branch to commit to (if save_to_repo)'
|
|
required: false
|
|
type: string
|
|
default: 'dev'
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Required secrets and variables (per environment):
|
|
#
|
|
# DEV ENVIRONMENT — secrets/variables:
|
|
# DEV_SSH_HOST — Dev server hostname
|
|
# DEV_SSH_PORT — SSH port (default: 22)
|
|
# DEV_SSH_USERNAME — SSH user
|
|
# DEV_SSH_KEY — SSH private key
|
|
# DEV_PULL_PATH — Remote install path (repo variable)
|
|
#
|
|
# DEMO ENVIRONMENT — secrets/variables:
|
|
# DEMO_FTP_HOST — Demo server hostname (reused from deploy)
|
|
# DEMO_FTP_PORT — SSH port (reused, default: 22)
|
|
# DEMO_FTP_USERNAME — SSH user (reused from deploy)
|
|
# DEMO_FTP_KEY — SSH key (reused from deploy)
|
|
# DEMO_FTP_PATH — Remote install path (repo variable)
|
|
#
|
|
# MySQL credentials are read automatically from:
|
|
# Joomla: configuration.php ($user, $password, $db)
|
|
# Dolibarr: conf/conf.php ($dolibarr_main_db_user, etc.)
|
|
# No MySQL secrets needed — credentials come from the remote config file.
|
|
# ──────────────────────────────────────────────────────────────
|
|
|
|
permissions:
|
|
contents: write
|
|
|
|
jobs:
|
|
export-mysql:
|
|
name: Export MySQL — ${{ inputs.environment }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Checkout
|
|
if: inputs.save_to_repo
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ inputs.branch }}
|
|
|
|
- name: Resolve environment config
|
|
id: env
|
|
run: |
|
|
ENV="${{ inputs.environment }}"
|
|
|
|
if [ "$ENV" = "dev" ]; then
|
|
HOST="${{ vars.DEV_SSH_HOST }}"
|
|
PORT="${{ vars.DEV_SSH_PORT || '22' }}"
|
|
USER="${{ vars.DEV_SSH_USERNAME }}"
|
|
DB="${{ inputs.database || vars.DEV_MYSQL_DATABASE }}"
|
|
MYSQL_USER="${{ vars.DEV_MYSQL_USER || 'root' }}"
|
|
elif [ "$ENV" = "demo" ]; then
|
|
HOST="${{ vars.DEMO_FTP_HOST }}"
|
|
PORT="${{ vars.DEMO_FTP_PORT || '22' }}"
|
|
USER="${{ vars.DEMO_FTP_USERNAME }}"
|
|
DB="${{ inputs.database || vars.DEMO_MYSQL_DATABASE }}"
|
|
MYSQL_USER="${{ vars.DEMO_MYSQL_USER || 'root' }}"
|
|
fi
|
|
|
|
MISSING=""
|
|
[ -z "$HOST" ] && MISSING="${MISSING} ${ENV^^}_SSH_HOST"
|
|
[ -z "$USER" ] && MISSING="${MISSING} ${ENV^^}_SSH_USERNAME"
|
|
[ -z "$DB" ] && MISSING="${MISSING} ${ENV^^}_MYSQL_DATABASE"
|
|
if [ -n "$MISSING" ]; then
|
|
echo "ERROR: Missing variables:${MISSING}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "host=${HOST}" >> $GITHUB_OUTPUT
|
|
echo "port=${PORT}" >> $GITHUB_OUTPUT
|
|
echo "user=${USER}" >> $GITHUB_OUTPUT
|
|
echo "database=${DB}" >> $GITHUB_OUTPUT
|
|
echo "mysql_user=${MYSQL_USER}" >> $GITHUB_OUTPUT
|
|
echo "Config OK — exporting ${DB} from ${USER}@${HOST}"
|
|
|
|
- name: Setup SSH
|
|
run: |
|
|
mkdir -p ~/.ssh
|
|
chmod 700 ~/.ssh
|
|
|
|
ENV="${{ inputs.environment }}"
|
|
if [ "$ENV" = "dev" ]; then
|
|
KEY="${{ secrets.DEV_SSH_KEY }}"
|
|
else
|
|
KEY="${{ secrets.DEMO_FTP_KEY }}"
|
|
fi
|
|
|
|
if [ -n "$KEY" ]; then
|
|
echo "$KEY" > ~/.ssh/export_key
|
|
chmod 600 ~/.ssh/export_key
|
|
else
|
|
echo "ERROR: No SSH key found for ${ENV} environment"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Host *" > ~/.ssh/config
|
|
echo " StrictHostKeyChecking no" >> ~/.ssh/config
|
|
echo " UserKnownHostsFile /dev/null" >> ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
- name: Export MySQL database
|
|
id: export
|
|
run: |
|
|
HOST="${{ steps.env.outputs.host }}"
|
|
PORT="${{ steps.env.outputs.port }}"
|
|
USER="${{ steps.env.outputs.user }}"
|
|
DB="${{ steps.env.outputs.database }}"
|
|
ENV="${{ inputs.environment }}"
|
|
CONFIG_PATH="${{ vars.DEV_PULL_PATH || vars.DEMO_FTP_PATH }}"
|
|
|
|
# Read MySQL credentials from the remote config file
|
|
# Joomla: configuration.php → $user, $password, $db
|
|
# Dolibarr: conf/conf.php → $dolibarr_main_db_user, $dolibarr_main_db_pass, $dolibarr_main_db_name
|
|
echo "Reading MySQL credentials from remote config file..."
|
|
|
|
CREDS=$(ssh -p "${PORT}" -i ~/.ssh/export_key "${USER}@${HOST}" bash << 'SSHEOF'
|
|
# Try Joomla configuration.php first
|
|
for cfg in "{{CONFIG_PATH}}/configuration.php" "/var/www/html/configuration.php" "$(find /var/www -name 'configuration.php' -maxdepth 3 2>/dev/null | head -1)"; do
|
|
if [ -f "$cfg" ]; then
|
|
DB_USER=$(php -r "include '$cfg'; echo \$user ?? '';")
|
|
DB_PASS=$(php -r "include '$cfg'; echo \$password ?? '';")
|
|
DB_NAME=$(php -r "include '$cfg'; echo \$db ?? '';")
|
|
DB_HOST=$(php -r "include '$cfg'; echo \$host ?? 'localhost';")
|
|
if [ -n "$DB_USER" ] && [ -n "$DB_NAME" ]; then
|
|
echo "TYPE=joomla"
|
|
echo "DB_USER=$DB_USER"
|
|
echo "DB_PASS=$DB_PASS"
|
|
echo "DB_NAME=$DB_NAME"
|
|
echo "DB_HOST=$DB_HOST"
|
|
exit 0
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Try Dolibarr conf/conf.php
|
|
for cfg in "{{CONFIG_PATH}}/conf/conf.php" "/var/www/html/conf/conf.php" "$(find /var/www -name 'conf.php' -path '*/conf/*' -maxdepth 4 2>/dev/null | head -1)"; do
|
|
if [ -f "$cfg" ]; then
|
|
DB_USER=$(php -r "include '$cfg'; echo \$dolibarr_main_db_user ?? '';")
|
|
DB_PASS=$(php -r "include '$cfg'; echo \$dolibarr_main_db_pass ?? '';")
|
|
DB_NAME=$(php -r "include '$cfg'; echo \$dolibarr_main_db_name ?? '';")
|
|
DB_HOST=$(php -r "include '$cfg'; echo \$dolibarr_main_db_host ?? 'localhost';")
|
|
if [ -n "$DB_USER" ] && [ -n "$DB_NAME" ]; then
|
|
echo "TYPE=dolibarr"
|
|
echo "DB_USER=$DB_USER"
|
|
echo "DB_PASS=$DB_PASS"
|
|
echo "DB_NAME=$DB_NAME"
|
|
echo "DB_HOST=$DB_HOST"
|
|
exit 0
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo "TYPE=not_found"
|
|
SSHEOF
|
|
)
|
|
|
|
PLATFORM=$(echo "$CREDS" | grep "^TYPE=" | cut -d= -f2)
|
|
MYSQL_USER=$(echo "$CREDS" | grep "^DB_USER=" | cut -d= -f2-)
|
|
MYSQL_PASS=$(echo "$CREDS" | grep "^DB_PASS=" | cut -d= -f2-)
|
|
DB_NAME=$(echo "$CREDS" | grep "^DB_NAME=" | cut -d= -f2-)
|
|
DB_HOST_REMOTE=$(echo "$CREDS" | grep "^DB_HOST=" | cut -d= -f2-)
|
|
|
|
if [ "$PLATFORM" = "not_found" ]; then
|
|
echo "ERROR: Could not find Joomla configuration.php or Dolibarr conf/conf.php on remote server"
|
|
exit 1
|
|
fi
|
|
|
|
# Override DB name if explicitly provided
|
|
[ -n "$DB" ] && DB_NAME="$DB"
|
|
|
|
echo "Platform: ${PLATFORM}"
|
|
echo "Database: ${DB_NAME} (user: ${MYSQL_USER}, host: ${DB_HOST_REMOTE})"
|
|
|
|
TIMESTAMP=$(date -u +%Y%m%d_%H%M%S)
|
|
FILENAME="${DB_NAME}_${ENV}_${TIMESTAMP}.sql"
|
|
|
|
echo "Exporting ${DB_NAME} from ${HOST}..."
|
|
|
|
# Run mysqldump over SSH using credentials from config file
|
|
ssh -p "${PORT}" -i ~/.ssh/export_key "${USER}@${HOST}" \
|
|
"mysqldump --single-transaction --no-tablespaces --routines --triggers \
|
|
-h ${DB_HOST_REMOTE} -u ${MYSQL_USER} -p'${MYSQL_PASS}' ${DB_NAME}" \
|
|
> "/tmp/${FILENAME}" 2>/tmp/mysqldump.err
|
|
|
|
if [ $? -ne 0 ]; then
|
|
echo "ERROR: mysqldump failed"
|
|
cat /tmp/mysqldump.err
|
|
exit 1
|
|
fi
|
|
|
|
SIZE=$(du -h "/tmp/${FILENAME}" | cut -f1)
|
|
LINES=$(wc -l < "/tmp/${FILENAME}")
|
|
echo "Export complete: ${FILENAME} (${SIZE}, ${LINES} lines)"
|
|
|
|
# ── Sanitize PII / credentials ──────────────────────────
|
|
echo "Sanitizing sensitive data..."
|
|
SANITIZED="/tmp/${FILENAME%.sql}_sanitized.sql"
|
|
cp "/tmp/${FILENAME}" "$SANITIZED"
|
|
|
|
# Joomla sanitization
|
|
# - Clear user passwords (set to bcrypt hash of 'sanitized')
|
|
# - Clear session data
|
|
# - Clear user email addresses (keep admin)
|
|
# - Clear reset tokens
|
|
BCRYPT_SANITIZED='$2y$10$sanitized.sanitized.sanitized.sanitized.sanitized.sa'
|
|
sed -i \
|
|
-e "s/\(VALUES([^)]*,'[^']*','\)\$2[ayb]\$[^']*\('/\1${BCRYPT_SANITIZED}'/g" \
|
|
"$SANITIZED"
|
|
|
|
# Joomla: clear sessions table content
|
|
sed -i '/INSERT INTO.*_session/d' "$SANITIZED"
|
|
|
|
# Joomla: sanitize user emails (replace with user{id}@sanitized.local)
|
|
python3 -c "
|
|
import re, sys
|
|
|
|
with open('$SANITIZED', encoding='utf-8', errors='replace') as f:
|
|
content = f.read()
|
|
|
|
# Joomla users table: sanitize emails but keep structure
|
|
# Pattern: email addresses in INSERT statements for user tables
|
|
content = re.sub(
|
|
r\"(['\\\"])([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})(['\\\"])\",
|
|
lambda m: m.group(1) + 'user@sanitized.local' + m.group(3)
|
|
if 'admin' not in m.group(2).lower() and 'moko' not in m.group(2).lower()
|
|
else m.group(0),
|
|
content
|
|
)
|
|
|
|
# Dolibarr: clear API keys, passwords, session tokens
|
|
content = re.sub(r\"'(api_key|pass_crypted|pass_temp|session_id|token)','[^']*'\", r\"'\1',''\", content)
|
|
|
|
# Dolibarr: clear LDAP passwords
|
|
content = re.sub(r\"'ldap_pass','[^']*'\", \"'ldap_pass',''\", content)
|
|
|
|
# Clear any raw passwords or tokens in conf values
|
|
content = re.sub(r\"'(smtp_pass|ftp_password|oauth_.*secret)','[^']*'\", r\"'\1','[SANITIZED]'\", content)
|
|
|
|
with open('$SANITIZED', 'w', encoding='utf-8') as f:
|
|
f.write(content)
|
|
|
|
print('Sanitization complete')
|
|
" 2>/dev/null || echo "Python sanitization skipped (no python3)"
|
|
|
|
mv "$SANITIZED" "/tmp/${FILENAME}"
|
|
echo "Sanitized: passwords, sessions, emails (admin/moko preserved)"
|
|
|
|
# Compress
|
|
gzip "/tmp/${FILENAME}"
|
|
GZ_SIZE=$(du -h "/tmp/${FILENAME}.gz" | cut -f1)
|
|
echo "Compressed: ${FILENAME}.gz (${GZ_SIZE})"
|
|
|
|
echo "filename=${FILENAME}" >> $GITHUB_OUTPUT
|
|
echo "gz_filename=${FILENAME}.gz" >> $GITHUB_OUTPUT
|
|
echo "size=${GZ_SIZE}" >> $GITHUB_OUTPUT
|
|
echo "lines=${LINES}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Commit to repo
|
|
if: inputs.save_to_repo && steps.export.outputs.filename != ''
|
|
run: |
|
|
mkdir -p sql/exports
|
|
cp "/tmp/${{ steps.export.outputs.gz_filename }}" "sql/exports/"
|
|
|
|
git config user.name "gitea-actions[bot]"
|
|
git config user.email "gitea-actions[bot]@noreply.git.mokoconsulting.tech"
|
|
git add sql/exports/
|
|
git commit -m "chore(db): export ${{ steps.env.outputs.database }} from ${{ inputs.environment }}
|
|
|
|
File: ${{ steps.export.outputs.gz_filename }}
|
|
Size: ${{ steps.export.outputs.size }}
|
|
Lines: ${{ steps.export.outputs.lines }}
|
|
Source: ${{ steps.env.outputs.user }}@${{ steps.env.outputs.host }}"
|
|
git push origin ${{ inputs.branch }}
|
|
echo "Committed to ${{ inputs.branch }}"
|
|
|
|
- name: Upload artifact
|
|
if: steps.export.outputs.filename != '' && github.server_url == 'https://github.com'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: mysql-export-${{ inputs.environment }}-${{ steps.env.outputs.database }}
|
|
path: /tmp/${{ steps.export.outputs.gz_filename }}
|
|
retention-days: 30
|
|
|
|
- name: Upload artifact (Gitea fallback)
|
|
if: steps.export.outputs.filename != '' && github.server_url != 'https://github.com'
|
|
run: |
|
|
echo "Artifact upload skipped (Gitea does not support actions/upload-artifact@v4)"
|
|
echo "Export file committed to branch: ${{ inputs.branch }}"
|
|
|
|
- name: Summary
|
|
run: |
|
|
echo "## MySQL Export — ${{ inputs.environment }}"
|
|
echo ""
|
|
echo "- Database: \`${{ steps.env.outputs.database }}\`"
|
|
echo "- Server: \`${{ steps.env.outputs.host }}\`"
|
|
echo "- File: \`${{ steps.export.outputs.gz_filename }}\`"
|
|
echo "- Size: ${{ steps.export.outputs.size }}"
|
|
echo "- Lines: ${{ steps.export.outputs.lines }}"
|
|
if [ "${{ inputs.save_to_repo }}" = "true" ]; then
|
|
echo "- Saved to: \`sql/exports/\` on branch \`${{ inputs.branch }}\`"
|
|
else
|
|
echo "- Available as workflow artifact (30 day retention)"
|
|
fi
|
|
|
|
- name: Cleanup
|
|
if: always()
|
|
run: rm -f ~/.ssh/export_key /tmp/*.sql /tmp/*.sql.gz
|