diff --git a/pipeline/cnpg-maintenance.sh b/pipeline/cnpg-maintenance.sh new file mode 100644 index 0000000..769439b --- /dev/null +++ b/pipeline/cnpg-maintenance.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ACTION="" +NAMESPACE="" +DRY_RUN=false + +usage() { + cat </dev/null 2>&1 || { + echo "Errore: kubectl non trovato" + exit 1 +} + +kubectl cnpg version >/dev/null 2>&1 || { + echo "Errore: plugin kubectl-cnpg non disponibile o non funzionante" + exit 1 +} + +# +# Recupero cluster CNPG +# +# label applicata dallo script per tracciare le ibernazioni proprie +SCRIPT_LABEL="cnpg-maintenance-hibernated" + +JSONPATH='{range .items[*]}{.metadata.namespace}{";"}{.metadata.name}{";"}{.status.readyInstances}{";"}{.metadata.annotations.cnpg\.io/hibernation}{";"}{.metadata.labels.cnpg-maintenance-hibernated}{"\n"}{end}' + +if [[ -n "$NAMESPACE" ]]; then + + CLUSTERS=$(kubectl get cluster.postgresql.cnpg.io \ + -n "$NAMESPACE" \ + -o jsonpath="$JSONPATH") + +else + + CLUSTERS=$(kubectl get cluster.postgresql.cnpg.io \ + -A \ + -o jsonpath="$JSONPATH") + +fi + +if [[ -z "$CLUSTERS" ]]; then + echo "Nessun cluster CNPG trovato." + exit 0 +fi + +echo +echo "========================================" +echo " CNPG BULK OPERATION" +echo "========================================" +echo "Azione: $ACTION" + +if [[ -n "$NAMESPACE" ]]; then + echo "Namespace: $NAMESPACE" +else + echo "Namespace: TUTTI" +fi + +echo "Dry-run: $DRY_RUN" +echo "========================================" +echo + +# +# Elaborazione cluster +# +while IFS=";" read -r NS CLUSTER READY_INSTANCES HIBERNATION_ANNOTATION SCRIPT_HIBERNATED; do + + [[ -z "$CLUSTER" ]] && continue + + if [[ "$ACTION" == "hibernate" ]]; then + # skip clusters con no ready instances (già ibernati o non attivi) + if [[ -z "$READY_INSTANCES" || "$READY_INSTANCES" -lt 1 ]]; then + echo "Cluster: $NS/$CLUSTER — SALTATO (readyInstances=${READY_INSTANCES:-0}, già ibernato o non attivo)" + echo + continue + fi + else + # resume: solo cluster ibernati da questo script (label presente) e con annotazione CNPG attiva + if [[ "$SCRIPT_HIBERNATED" != "true" || "$HIBERNATION_ANNOTATION" != "on" ]]; then + echo "Cluster: $NS/$CLUSTER — SALTATO (non ibernato da questo script)" + echo + continue + fi + fi + + echo "Cluster: $NS/$CLUSTER (readyInstances=${READY_INSTANCES:-0}, hibernation=${HIBERNATION_ANNOTATION:-none}, script-label=${SCRIPT_HIBERNATED:-none})" + + if [[ "$ACTION" == "hibernate" ]]; then + + CMD=( + kubectl + cnpg + hibernate + on + "$CLUSTER" + -n + "$NS" + ) + + else + + CMD=( + kubectl + cnpg + hibernate + off + "$CLUSTER" + -n + "$NS" + ) + + fi + + if [[ "$DRY_RUN" == true ]]; then + + echo "[DRY-RUN] ${CMD[*]}" + if [[ "$ACTION" == "hibernate" ]]; then + echo "[DRY-RUN] kubectl label cluster.postgresql.cnpg.io $CLUSTER -n $NS ${SCRIPT_LABEL}=true" + else + echo "[DRY-RUN] kubectl label cluster.postgresql.cnpg.io $CLUSTER -n $NS ${SCRIPT_LABEL}-" + fi + + else + + echo "Esecuzione: ${CMD[*]}" + + if "${CMD[@]}"; then + echo "OK: $NS/$CLUSTER" + if [[ "$ACTION" == "hibernate" ]]; then + kubectl label cluster.postgresql.cnpg.io "$CLUSTER" -n "$NS" "${SCRIPT_LABEL}=true" --overwrite + else + kubectl label cluster.postgresql.cnpg.io "$CLUSTER" -n "$NS" "${SCRIPT_LABEL}-" + fi + else + echo "ERRORE: $NS/$CLUSTER" + fi + + fi + + echo + +done <<< "$CLUSTERS" + +echo "========================================" +echo "Operazione completata" +echo "========================================" \ No newline at end of file diff --git a/st2idc/README-clone-transform.md b/st2idc/README-clone-transform.md new file mode 100644 index 0000000..86d8b13 --- /dev/null +++ b/st2idc/README-clone-transform.md @@ -0,0 +1,287 @@ +# Clone and Transform Script + +Script bash per clonare un repository GitLab usando API e token privato, con sostituzione automatica di stringhe nei file. + +## Prerequisiti + +- `bash` (4.0+) +- `git` +- `curl` (opzionale, per verificare l'accesso) +- Token privato GitLab con almeno scope `api` o `read_repository` + +## Preparazione + +### 1. Creare il file `transform.txt` + +Crea un file `transform.txt` nella stessa directory dello script con il formato: + +```txt += += +... +``` + +**Esempio di transform.txt:** + +```txt +# Configurazioni di sviluppo verso produzione +DATABASE_HOST=localhost=DATABASE_HOST=prod.example.com +DATABASE_PORT=5432=DATABASE_PORT=5432 +API_ENDPOINT=http://localhost:8080=API_ENDPOINT=https://api.prod.com +ENVIRONMENT=development=ENVIRONMENT=production +LOG_LEVEL=debug=LOG_LEVEL=info +SECRET_KEY=dev-secret=SECRET_KEY=prod-secret-key-xyz +``` + +### 2. Ottenere un token GitLab + +1. Accedi a GitLab +2. Vai a **Profile → Access Tokens** +3. Crea un nuovo token con scopes: + - `api` - accesso completo + - oppure `read_repository` - solo lettura +4. Copia il token (non sarà più visibile dopo) + +### 3. Rendere eseguibile lo script + +```bash +chmod +x clone-and-transform.sh +``` + +## Utilizzo + +### Sintassi base + +```bash +./clone-and-transform.sh -u -t -r -d +``` + +### Parametri + +| Parametro | Breve | Descrizione | Esempio | +|-----------|-------|-------------|---------| +| `--url` | `-u` | URL base di GitLab | `https://gitlab.com` | +| `--token` | `-t` | Token di accesso privato | `glpat-xxxxx` | +| `--repo` | `-r` | Percorso del repository | `mygroup/myproject` | +| `--dest` | `-d` | Directory di destinazione | `./cloned-repo` | +| `--transform` | `-f` | Path del file transform.txt | `./transform.txt` (default) | +| `--help` | `-h` | Mostra l'aiuto | - | + +### Esempi di utilizzo + +**Esempio 1: Clone con trasformazioni usando file default** + +```bash +./clone-and-transform.sh \ + -u https://gitlab.com \ + -t glpat-xxxxxxxxxxxx \ + -r mygroup/myproject \ + -d ./my-cloned-repo +``` + +**Esempio 2: Clone con file transform.txt personalizzato** + +```bash +./clone-and-transform.sh \ + -u https://gitlab.internal.com \ + -t glpat-xxxxxxxxxxxx \ + -r company/backend \ + -d ./backend-clone \ + -f ./custom-transform.txt +``` + +**Esempio 3: Clone da GitLab Enterprise** + +```bash +./clone-and-transform.sh \ + -u https://gitlab.mycompany.com \ + -t glpat-xxxxxxxxxxxx \ + -r products/idcidp \ + -d ./idcidp-prod +``` + +## Come funziona lo script + +1. **Validazione**: Verifica i parametri e l'esistenza di `transform.txt` +2. **Clone**: Clona il repository usando autenticazione OAuth2 +3. **Scoperta directory**: Identifica directory `kubernetes`, `containers`, `conf` +4. **Trasformazione**: Per ogni file trovato applica le sostituzioni definite in `transform.txt` +5. **Report**: Mostra il number di file modificati e suggerisce i prossimi passi + +### Directory elaborate + +Lo script elabora automaticamente questi percorsi: +- `kubernetes/` +- `containers/` +- `conf/` + +Se una directory non esiste, viene saltata con un avvertimento. + +### File binari + +I file binari (immagini, archivi, ecc.) vengono automaticamente saltati per evitare corruzioni. + +## Output esempio + +``` +[INFO] Starting GitLab repository clone and transformation... +[INFO] GitLab URL: https://gitlab.com +[INFO] Repository: mygroup/myproject +[INFO] Destination: ./my-cloned-repo +[INFO] Transform file: ./transform.txt +[INFO] Cloning repository... +[INFO] Repository cloned successfully +[INFO] Found directory: kubernetes +[INFO] Found directory: containers +[INFO] Applying transformations from ./transform.txt... +[INFO] Processing directory: kubernetes +[INFO] Modified: ./my-cloned-repo/kubernetes/deployment.yaml +[INFO] Modified: ./my-cloned-repo/kubernetes/service.yaml +[INFO] Processing directory: containers +[INFO] ========== TRANSFORMATION SUMMARY ========== +Total files processed: 12 +Files modified: 5 +[INFO] ========================================== +[INFO] Transformation completed successfully! + +Next steps: + 1. Review changes: cd my-cloned-repo && git diff + 2. Commit changes: git add . && git commit -m 'Apply transformations' + 3. Push changes: git push +``` + +## Verifica delle modifiche + +Dopo l'esecuzione, è consigliato verificare le modifiche: + +```bash +cd ./my-cloned-repo +git diff +``` + +Per vedere solo i file modificati: + +```bash +git status +``` + +## Commit e Push + +Se le modifiche sono corrette: + +```bash +git add . +git commit -m "Apply environment transformations" +git push +``` + +## Troubleshooting + +### Errore: "Failed to clone repository" + +**Causa**: Token non valido o senza permessi + +**Soluzione**: +1. Verifica il token nel profilo GitLab +2. Assicurati che il token abbia scope `api` +3. Verifica che il repository sia accessibile con il token + +### Errore: "Transform file not found" + +**Causa**: Il file `transform.txt` non esiste + +**Soluzione**: +1. Crea il file `transform.txt` nella directory dove esegui lo script +2. Usa l'opzione `-f` per specificare un percorso personalizzato + +### Nessun file modificato + +**Causa**: Le stringhe in `transform.txt` non corrispondono ai file + +**Soluzione**: +1. Verifica il contenuto dei file nella directory clonata +2. Assicurati che le stringhe in `transform.txt` siano corrette +3. Usa pattern più generici se necessario + +### Errore di escaping in stringhe complesse + +Se le stringhe contengono caratteri speciali (barre, ampersand, etc.): + +**Lo script gestisce automaticamente l'escaping**, ma se riscontri problemi: +1. Usa sequenze di escape nel file transform.txt +2. Ad esempio, per una barra inversa: `path\\old=path\\new` + +## Sicurezza + +### Best practices + +1. **Non** commettere il token nel repository +2. Usa variabili d'ambiente per il token: + ```bash + ./clone-and-transform.sh \ + -u https://gitlab.com \ + -t $GITLAB_TOKEN \ + -r mygroup/myproject \ + -d ./clone + ``` + +3. Usa file `.gitignore` per `transform.txt` se contiene dati sensibili: + ```bash + echo "transform.txt" >> .gitignore + ``` + +4. Revoca il token dopo l'uso se è ad uso singolo + +## Opzioni avanzate + +### Usare con script di automazione + +```bash +#!/bin/bash +export GITLAB_TOKEN="glpat-xxxxx" +export GITLAB_URL="https://gitlab.internal.com" + +./clone-and-transform.sh \ + -u $GITLAB_URL \ + -t $GITLAB_TOKEN \ + -r company/backend \ + -d ./backend-clone +``` + +### Variazione per ambienti multipli + +Crea diversi file di trasformazione: +- `transform-dev.txt` +- `transform-qa.txt` +- `transform-prod.txt` + +```bash +# Per development +./clone-and-transform.sh \ + -u https://gitlab.com \ + -t $GITLAB_TOKEN \ + -r mygroup/myproject \ + -d ./clone-dev \ + -f transform-dev.txt + +# Per production +./clone-and-transform.sh \ + -u https://gitlab.com \ + -t $GITLAB_TOKEN \ + -r mygroup/myproject \ + -d ./clone-prod \ + -f transform-prod.txt +``` + +## Changelog + +### v1.0 (Initial Release) +- Clone repository con token privato +- Applicazione di trasformazioni su directory specifiche +- Supporto per file UTF-8 +- Skip automatico file binari +- Output colorato e dettagliato + +## Licenza + +MIT diff --git a/st2idc/backupPG.sh b/st2idc/backupPG.sh new file mode 100644 index 0000000..291bffd --- /dev/null +++ b/st2idc/backupPG.sh @@ -0,0 +1,101 @@ +#!/bin/bash +set -Eeuo pipefail + +# Configuration +SOURCE_ENV="${1:-dbdev}" +DBNAME="${2:-geco}" +POD_NAME="${3:-postgresql-1-postgresql-0}" +CONTAINER_NAME="${4:-postgresql-server}" +BACKUP_REMOTE_DIR="${5:-/shared/backup}" +BACKUP_LOCAL_DIR="${6:-.}" +COMPRESSION="${7:-true}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Logging function +log() { + echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $*" +} + +error() { + echo -e "${RED}[$(date '+%Y-%m-%d %H:%M:%S')] ERROR:${NC} $*" >&2 + exit 1 +} + +warn() { + echo -e "${YELLOW}[$(date '+%Y-%m-%d %H:%M:%S')] WARN:${NC} $*" +} + +# Determine backup file extension +if [ "${COMPRESSION}" = "true" ]; then + BACKUP_FILE="${DBNAME}-${SOURCE_ENV}-$(date +%Y%m%d-%H%M%S).backup.gz" + DUMP_FORMAT="c" +else + BACKUP_FILE="${DBNAME}-${SOURCE_ENV}-$(date +%Y%m%d-%H%M%S).backup" + DUMP_FORMAT="c" +fi + +log "Inizio backup database PostgreSQL" +log "Namespace: ${SOURCE_ENV}" +log "Pod: ${POD_NAME}" +log "Database: ${DBNAME}" +log "File di backup: ${BACKUP_FILE}" + +# Check if namespace exists +if ! kubectl get namespace "${SOURCE_ENV}" &>/dev/null; then + error "Namespace '${SOURCE_ENV}' non trovato" +fi + +# Check if pod exists +if ! kubectl get pod "${POD_NAME}" -n "${SOURCE_ENV}" &>/dev/null; then + error "Pod '${POD_NAME}' non trovato nel namespace '${SOURCE_ENV}'" +fi + +# Check if pod is running +POD_STATUS=$(kubectl get pod "${POD_NAME}" -n "${SOURCE_ENV}" -o jsonpath='{.status.phase}') +if [ "${POD_STATUS}" != "Running" ]; then + error "Pod '${POD_NAME}' non è in stato Running (stato attuale: ${POD_STATUS})" +fi + +log "Creazione backup sul pod..." + +# Create backup on remote pod +if [ "${COMPRESSION}" = "true" ]; then + kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${SOURCE_ENV}" \ + -- pg_dump -U postgres -d "${DBNAME}" -F "${DUMP_FORMAT}" -b -v | gzip > "${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" || error "Backup fallito" +else + kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${SOURCE_ENV}" \ + -- pg_dump -U postgres -d "${DBNAME}" -F "${DUMP_FORMAT}" -b -v > "${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" || error "Backup fallito" +fi + +log "Backup completato" + +# Verify file exists and has content +if [ ! -f "${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" ]; then + error "File di backup non trovato: ${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" +fi + +FILE_SIZE=$(du -h "${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" | cut -f1) +if [ -z "${FILE_SIZE}" ] || [ "${FILE_SIZE}" = "0" ]; then + error "File di backup vuoto o non valido" +fi + +log "✅ Backup completato con successo" +log "Percorso: ${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" +log "Dimensione: ${FILE_SIZE}" + +log "Inizio trasferimento file di backup su server remoto..." +if sshpass -p "POC-25_sts" scp "${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" barucci@10.20.1.101:/home/barucci/froms/; then + log "✅ File trasferito con successo su 10.20.1.101" + log "Destinazione: barucci@10.20.1.101:/home/barucci/froms/${BACKUP_FILE}" +else + warn "Trasferimento file fallito, ma backup locale è disponibile in ${BACKUP_LOCAL_DIR}/${BACKUP_FILE}" +fi \ No newline at end of file diff --git a/st2idc/clone-and-transform.sh b/st2idc/clone-and-transform.sh new file mode 100644 index 0000000..a4c495b --- /dev/null +++ b/st2idc/clone-and-transform.sh @@ -0,0 +1,773 @@ +#!/bin/bash + +################################################################################ +# Script: clone-and-transform.sh +# Description: Clone GitLab repository using private token and apply +# string substitutions from transform.env +# Usage: ./clone-and-transform.sh +################################################################################ + +set -e # Exit on error + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Function to print colored output +print_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +# Function to display usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +OPTIONS: + -u, --url GitLab API base URL (e.g., https://gitlab.com) + -t, --token Private access token + -r, --repo Repository path (e.g., group/project) + -d, --dest Destination directory for clone / transformation source + -f, --transform Path to transform.env file (default: ./transform.env) + --gitea-repo Repository path to clone from Gitea (default: same as --repo) + -transformation Skip GitLab clone; run Gitea clone + transform + commit/push + -h, --help Show this help message + +Example: + $0 -u https://gitlab.com -t glpat-xxxxx -r mygroup/myproject -d ./cloned-repo + $0 -transformation -d ./existing-repo --gitea-repo mygroup/myproject -f ./transform.env +EOF + exit 1 +} + +# Default values +GITLAB_URL="" +PRIVATE_TOKEN="" +REPO_PATH="" +DEST_DIR="" +TRANSFORM_FILE="./transform.env" +GITEA_URL="https://git.italiadatacenter.com" +GITEA_TOKEN="65fc5f07365bd3f89c5b9ddca3d2d2e7b3777a88" +GITEA_REPO_PATH="" +GITEA_DEST_DIR="target-repo" +TRANSFORMATION_ONLY=false + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -u|--url) + GITLAB_URL="$2" + shift 2 + ;; + -t|--token) + PRIVATE_TOKEN="$2" + shift 2 + ;; + -r|--repo) + REPO_PATH="$2" + shift 2 + ;; + -d|--dest) + DEST_DIR="$2" + shift 2 + ;; + -f|--transform) + TRANSFORM_FILE="$2" + shift 2 + ;; + --gitea-repo) + GITEA_REPO_PATH="$2" + shift 2 + ;; + -transformation) + TRANSFORMATION_ONLY=true + shift + ;; + -h|--help) + usage + ;; + *) + print_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate required parameters +if [[ "$TRANSFORMATION_ONLY" == true ]]; then + if [[ -z "$DEST_DIR" ]]; then + print_error "Missing required parameter: --dest" + usage + fi +else + if [[ -z "$GITLAB_URL" ]] || [[ -z "$PRIVATE_TOKEN" ]] || [[ -z "$REPO_PATH" ]] || [[ -z "$DEST_DIR" ]]; then + print_error "Missing required parameters" + usage + fi +fi + +# Check if transform.env exists +if [[ ! -f "$TRANSFORM_FILE" ]]; then + print_error "Transform file not found: $TRANSFORM_FILE" + exit 1 +fi + +# Validate transform.env format +if ! grep -Eq '^[[:space:]]*[^#[:space:]].*===.*$' "$TRANSFORM_FILE"; then + print_error "Transform file format invalid. Expected: ===" + exit 1 +fi + +if [[ "$TRANSFORMATION_ONLY" == true ]]; then + print_info "Mode: transformation only (GitLab clone skipped)" +else + print_info "Starting GitLab repository clone and transformation..." + print_info "GitLab URL: $GITLAB_URL" + print_info "Repository: $REPO_PATH" +fi +print_info "Destination: $DEST_DIR" +print_info "Transform file: $TRANSFORM_FILE" +print_info "Gitea instance: $GITEA_URL" +print_info "Gitea destination: $GITEA_DEST_DIR" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HTTPROUTE_TEMPLATE_FILE="$SCRIPT_DIR/httproute.yaml" + +if [[ "$TRANSFORMATION_ONLY" == false ]]; then +# Build repository URLs +BASE_URL="${GITLAB_URL%/}" +if [[ ! "$BASE_URL" =~ ^https?:// ]]; then + print_error "GitLab URL must start with http:// or https://" + exit 1 +fi + +TARGET_REPO_URL="${BASE_URL}/${REPO_PATH}.git" +AUTH_REPO_URL="$(printf '%s\n' "$TARGET_REPO_URL" | sed -E "s#^(https?://)#\\1oauth2:${PRIVATE_TOKEN}@#")" + +# Always use a fresh clone to avoid stale/outdated repository state +if [[ -e "$DEST_DIR" ]]; then + if [[ -z "$DEST_DIR" || "$DEST_DIR" == "/" || "$DEST_DIR" == "." ]]; then + print_error "Refusing to remove unsafe destination path: $DEST_DIR" + exit 1 + fi + print_info "Destination already exists, removing it before clone: $DEST_DIR" + rm -rf -- "$DEST_DIR" +fi + +print_info "Cloning repository..." +print_info "Running: git clone $(printf '%s' "$AUTH_REPO_URL" | sed -E 's#(https?://)[^@]+@#\1***@#') $DEST_DIR" +GIT_TERMINAL_PROMPT=0 git clone --quiet "$AUTH_REPO_URL" "$DEST_DIR" 2>/dev/null || { + print_error "Failed to clone repository" + print_info "Ensure your token has 'api' or 'read_repository' scope" + exit 1 +} +print_info "Repository cloned successfully" +fi # end TRANSFORMATION_ONLY skip + +# Clone additional repository from Gitea into target-repo +if [[ -z "$GITEA_REPO_PATH" ]]; then + GITEA_REPO_PATH="$REPO_PATH" +fi + +GITEA_BASE_URL="${GITEA_URL%/}" +if [[ ! "$GITEA_BASE_URL" =~ ^https?:// ]]; then + print_error "Gitea URL must start with http:// or https://" + exit 1 +fi + +GITEA_TARGET_REPO_URL="${GITEA_BASE_URL}/${GITEA_REPO_PATH}.git" +GITEA_AUTH_REPO_URL="$(printf '%s\n' "$GITEA_TARGET_REPO_URL" | sed -E "s#^(https?://)#\\1${GITEA_TOKEN}@#")" + +if [[ -e "$GITEA_DEST_DIR" ]]; then + if [[ -z "$GITEA_DEST_DIR" || "$GITEA_DEST_DIR" == "/" || "$GITEA_DEST_DIR" == "." ]]; then + print_error "Refusing to remove unsafe destination path: $GITEA_DEST_DIR" + exit 1 + fi + print_info "Gitea destination already exists, removing it before clone: $GITEA_DEST_DIR" + rm -rf -- "$GITEA_DEST_DIR" +fi + +print_info "Cloning additional repository from Gitea..." +print_info "Running: git clone $(printf '%s' "$GITEA_AUTH_REPO_URL" | sed -E 's#(https?://)[^@]+@#\1***@#') $GITEA_DEST_DIR" +GIT_TERMINAL_PROMPT=0 git clone --quiet "$GITEA_AUTH_REPO_URL" "$GITEA_DEST_DIR" 2>/dev/null || { + print_error "Failed to clone additional Gitea repository" + print_info "Check Gitea token permissions and repository path" + exit 1 +} +print_info "Additional Gitea repository cloned successfully" + +# In transformation-only mode, seed DEST_DIR from the freshly cloned Gitea repo +if [[ "$TRANSFORMATION_ONLY" == true ]]; then + if [[ -e "$DEST_DIR" ]]; then + if [[ -z "$DEST_DIR" || "$DEST_DIR" == "/" || "$DEST_DIR" == "." ]]; then + print_error "Refusing to remove unsafe destination path: $DEST_DIR" + exit 1 + fi + print_info "Removing existing DEST_DIR before seeding from Gitea: $DEST_DIR" + rm -rf -- "$DEST_DIR" + fi + print_info "Copying Gitea clone ($GITEA_DEST_DIR) -> $DEST_DIR ..." + cp -R "$GITEA_DEST_DIR" "$DEST_DIR" + print_info "Copy completed" +fi + +# Check if target directories exist +DIRS_TO_PROCESS=("kubernetes" "container" "conf" "envs") +FOUND_DIRS=() + +for dir in "${DIRS_TO_PROCESS[@]}"; do + if [[ -d "$DEST_DIR/$dir" ]]; then + FOUND_DIRS+=("$dir") + print_info "Found directory: $dir" + fi +done + +# Also search for envs directories nested in subdirectories +while IFS= read -r envs_abs; do + envs_rel="${envs_abs#${DEST_DIR}/}" + # Skip top-level envs (already handled above) + if [[ "$envs_rel" != "envs" ]]; then + FOUND_DIRS+=("$envs_rel") + print_info "Found nested envs directory: $envs_rel" + fi +done < <(find "$DEST_DIR" -mindepth 2 -type d -name "envs" 2>/dev/null) + +if [[ ${#FOUND_DIRS[@]} -eq 0 ]]; then + print_warning "No target directories (kubernetes/container/conf/envs) found in cloned repository" +fi + +# Function to apply transformations to a file +apply_transformations() { + local file="$1" + local temp_file + local changes=0 + + temp_file="$(mktemp "${file}.tmp.XXXXXX")" + + # Create a copy + cp "$file" "$temp_file" + + # Apply each transformation from transform.env (format: old_string===new_string) + while IFS= read -r line || [[ -n "$line" ]]; do + # Skip empty lines and comments + [[ -z "$line" ]] && continue + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ "$line" != *"==="* ]] && continue + + local old_string new_string + old_string="${line%%===*}" + new_string="${line#*===}" + + # Trim whitespace + old_string=$(echo "$old_string" | xargs) + new_string=$(echo "$new_string" | xargs) + + # Check if substitution would occur + if grep -Fq -- "$old_string" "$temp_file" 2>/dev/null; then + # Use sed with proper escaping for replacement + local old_escaped new_escaped + old_escaped=$(printf '%s\n' "$old_string" | sed 's/[&|\\]/\\&/g') + new_escaped=$(printf '%s\n' "$new_string" | sed 's/[&|\\]/\\&/g') + sed -i "s|${old_escaped}|${new_escaped}|g" "$temp_file" + changes=$((changes + 1)) + fi + done < "$TRANSFORM_FILE" + + # Compare and update if changes were made + if ! diff -q "$file" "$temp_file" > /dev/null 2>&1; then + mv "$temp_file" "$file" + return 0 + else + rm "$temp_file" + return 1 + fi +} + +# Function to apply automatic placeholder transformation: %string% -> +apply_placeholder_transformations() { + local file="$1" + local temp_file + + temp_file="$(mktemp "${file}.tmp.XXXXXX")" + cp "$file" "$temp_file" + + # Convert placeholders enclosed in %...% to <...> + sed -i -E 's/%([^%]+)%/<\1>/g' "$temp_file" + + if ! diff -q "$file" "$temp_file" > /dev/null 2>&1; then + mv "$temp_file" "$file" + return 0 + else + rm "$temp_file" + return 1 + fi +} + +# Apply transformations to files in target directories +print_info "Applying transformations from $TRANSFORM_FILE..." + +total_files=0 +modified_files=0 + +for dir in "${FOUND_DIRS[@]}"; do + target_path="$DEST_DIR/$dir" + print_info "Processing directory: $dir" + + # Find all files in the directory and its subdirectories + while IFS= read -r file; do + total_files=$((total_files + 1)) + + # Skip binary files + if file "$file" | grep -q "binary"; then + print_warning "Skipping binary file: $file" + continue + fi + + if apply_transformations "$file"; then + print_info " Modified: $file" + modified_files=$((modified_files + 1)) + fi + done < <(find "$target_path" -type f 2>/dev/null) +done + +# Apply automatic %string% -> transformation to all files in target directory +print_info "Applying automatic placeholder transformation (%...% -> <...>) in $DEST_DIR..." + +placeholder_total_files=0 +placeholder_modified_files=0 + +while IFS= read -r file; do + placeholder_total_files=$((placeholder_total_files + 1)) + + # Skip binary files + if file "$file" | grep -q "binary"; then + print_warning "Skipping binary file: $file" + continue + fi + + if apply_placeholder_transformations "$file"; then + print_info " Placeholder updated: $file" + placeholder_modified_files=$((placeholder_modified_files + 1)) + fi +done < <(find "$DEST_DIR" -type f 2>/dev/null) + +# Post-transformation directory operations on DEST_DIR +if [[ "$TRANSFORMATION_ONLY" == false ]]; then +print_info "Applying post-transformation directory operations on $DEST_DIR..." + +# 1) Rename container -> containers +if [[ -d "$DEST_DIR/container" ]]; then + mv -- "$DEST_DIR/container" "$DEST_DIR/containers" + print_info "Renamed: container -> containers" +else + print_warning "Directory not found, skipping rename: $DEST_DIR/container" +fi + +# 2) Rename conf -> env (fail if env already exists) +if [[ -d "$DEST_DIR/conf" ]]; then + if [[ -e "$DEST_DIR/env" ]]; then + print_error "Cannot rename conf -> env: destination $DEST_DIR/env already exists" + exit 1 + fi + mv -- "$DEST_DIR/conf" "$DEST_DIR/env" + print_info "Renamed: conf -> env" +else + print_warning "Directory not found, skipping rename: $DEST_DIR/conf" +fi + +# 3) Remove .git directory +if [[ -d "$DEST_DIR/.git" ]]; then + rm -rf -- "$DEST_DIR/.git" + print_info "Removed: $DEST_DIR/.git" +else + print_warning "Directory not found, skipping removal: $DEST_DIR/.git" +fi + +# 4) Remove .gitlab-ci.yml +if [[ -f "$DEST_DIR/.gitlab-ci.yml" ]]; then + rm -f -- "$DEST_DIR/.gitlab-ci.yml" + print_info "Removed: $DEST_DIR/.gitlab-ci.yml" +else + print_warning "File not found, skipping removal: $DEST_DIR/.gitlab-ci.yml" +fi + +# Summary +echo "" +print_info "========== TRANSFORMATION SUMMARY ==========" +echo "Total files processed: $total_files" +echo "Files modified: $modified_files" +echo "Placeholder files scanned: $placeholder_total_files" +echo "Placeholder files modified: $placeholder_modified_files" +print_info "==========================================" + +if [[ $modified_files -gt 0 ]]; then + print_info "Transformation completed successfully!" +else + print_warning "No files were modified. Check transform.env contents and patterns." +fi + +# Rename configuration.txt -> values.env under env directory +if [[ -d "$DEST_DIR/env" ]]; then + print_info "Rinomino configuration.txt -> values.env in $DEST_DIR/env ..." + while IFS= read -r f; do + target_path="$(dirname "$f")/values.env" + mv -- "$f" "$target_path" + print_info " Renamed: $f -> $target_path" + done < <(find "$DEST_DIR/env" -type f -name "configuration.txt") +else + print_warning "Directory $DEST_DIR/env non trovata, skip rename configuration.txt" +fi + +# Move env/configuration.env to DEST_DIR root as properties.env +if [[ -f "$DEST_DIR/env/configuration.env" ]]; then + if [[ -f "$DEST_DIR/properties.env" ]]; then + rm -f -- "$DEST_DIR/properties.env" + print_info "Removed existing file: $DEST_DIR/properties.env" + fi + mv -- "$DEST_DIR/env/configuration.env" "$DEST_DIR/properties.env" + print_info "Moved: $DEST_DIR/env/configuration.env -> $DEST_DIR/properties.env" +else + print_warning "File non trovato, skip move: $DEST_DIR/env/configuration.env" +fi + +# If an Ingress resource exists in kubernetes YAMLs, generate httproute.yaml from template +# and remove Ingress resource definitions from manifests. +K8S_DIR="$DEST_DIR/kubernetes" +if [[ -d "$K8S_DIR" ]]; then + print_info "Checking Kubernetes manifests for Ingress resources in $K8S_DIR ..." + + ingress_found=false + service_name="" + ingress_service_name="" + ingress_port="" + + while IFS= read -r -d '' k8s_file; do + if [[ -z "$service_name" ]]; then + service_name="$(awk ' + /^[[:space:]]*kind:[[:space:]]*Service([[:space:]]|$)/ { in_service=1; next } + in_service && /^[[:space:]]*kind:[[:space:]]*/ { in_service=0 } + in_service && /^[[:space:]]*name:[[:space:]]*/ { + value=$0 + sub(/^[[:space:]]*name:[[:space:]]*/, "", value) + gsub(/[[:space:]]+$/, "", value) + print value + exit + } + ' "$k8s_file")" + fi + + if grep -Eq '^[[:space:]]*kind:[[:space:]]*Ingress([[:space:]]|$)' "$k8s_file"; then + ingress_found=true + + if [[ -z "$ingress_port" ]]; then + ingress_port="$(awk ' + /^[[:space:]]*kind:[[:space:]]*Ingress([[:space:]]|$)/ { in_ingress=1; next } + in_ingress && /^[[:space:]]*kind:[[:space:]]*/ { in_ingress=0 } + in_ingress && /^[[:space:]]*number:[[:space:]]*[0-9]+/ { + value=$0 + sub(/.*number:[[:space:]]*/, "", value) + gsub(/[^0-9].*$/, "", value) + print value + exit + } + ' "$k8s_file")" + fi + + if [[ -z "$ingress_service_name" ]]; then + ingress_service_name="$(awk ' + /^[[:space:]]*kind:[[:space:]]*Ingress([[:space:]]|$)/ { in_ingress=1; next } + in_ingress && /^[[:space:]]*kind:[[:space:]]*/ { in_ingress=0 } + in_ingress && /^[[:space:]]*name:[[:space:]]*/ { + value=$0 + sub(/^[[:space:]]*name:[[:space:]]*/, "", value) + gsub(/[[:space:]]+$/, "", value) + print value + exit + } + ' "$k8s_file")" + fi + fi + done < <(find "$K8S_DIR" -type f \( -name "*.yaml" -o -name "*.yml" \) -print0) + + if [[ "$ingress_found" == true ]]; then + if [[ ! -f "$HTTPROUTE_TEMPLATE_FILE" ]]; then + print_error "HTTPRoute template not found: $HTTPROUTE_TEMPLATE_FILE" + exit 1 + fi + + if [[ -z "$service_name" ]]; then + service_name="$ingress_service_name" + fi + + if [[ -z "$service_name" ]]; then + print_error "Cannot create HTTPRoute: service name not found in Kubernetes YAML" + exit 1 + fi + + if [[ -z "$ingress_port" ]]; then + print_error "Cannot create HTTPRoute: ingress port number not found in Kubernetes YAML" + exit 1 + fi + + httproute_output="$K8S_DIR/httproute.yaml" + sed -e "s||$service_name|g" -e "s||$ingress_port|g" "$HTTPROUTE_TEMPLATE_FILE" > "$httproute_output" + print_info "Created HTTPRoute file: $httproute_output" + + while IFS= read -r -d '' k8s_file; do + temp_k8s_file="$(mktemp "${k8s_file}.tmp.XXXXXX")" + awk ' + function flush_doc() { + if (!doc_started) { + return + } + + if (!doc_is_ingress) { + if (output_count > 0) { + printf "---\n" + } + printf "%s", doc + output_count++ + } + + doc = "" + doc_started = 0 + doc_is_ingress = 0 + } + + /^[[:space:]]*---[[:space:]]*$/ { + flush_doc() + next + } + + { + doc_started = 1 + doc = doc $0 "\n" + if ($0 ~ /^[[:space:]]*kind:[[:space:]]*Ingress([[:space:]]|$)/) { + doc_is_ingress = 1 + } + } + + END { + flush_doc() + } + ' "$k8s_file" > "$temp_k8s_file" + + if [[ ! -s "$temp_k8s_file" ]]; then + rm -f -- "$k8s_file" + rm -f -- "$temp_k8s_file" + print_info "Removed ingress-only manifest file: $k8s_file" + else + mv -- "$temp_k8s_file" "$k8s_file" + fi + done < <(find "$K8S_DIR" -type f \( -name "*.yaml" -o -name "*.yml" \) -print0) + + print_info "Ingress resource definitions removed from Kubernetes manifests" + else + print_info "No Ingress resource found in Kubernetes manifests" + fi + + # If a PersistentVolume resource exists in kubernetes YAMLs, + # remove PersistentVolume resource definitions from manifests. + pv_found=false + while IFS= read -r -d '' k8s_file; do + if grep -Eq '^[[:space:]]*kind:[[:space:]]*PersistentVolume([[:space:]]|$)' "$k8s_file"; then + pv_found=true + break + fi + done < <(find "$K8S_DIR" -type f \( -name "*.yaml" -o -name "*.yml" \) -print0) + + if [[ "$pv_found" == true ]]; then + print_info "PersistentVolume resources found, removing from Kubernetes manifests ..." + + while IFS= read -r -d '' k8s_file; do + temp_k8s_file="$(mktemp "${k8s_file}.tmp.XXXXXX")" + awk ' + function flush_doc() { + if (!doc_started) { + return + } + + if (!doc_is_pv) { + if (output_count > 0) { + printf "---\n" + } + printf "%s", doc + output_count++ + } + + doc = "" + doc_started = 0 + doc_is_pv = 0 + } + + /^[[:space:]]*---[[:space:]]*$/ { + flush_doc() + next + } + + { + doc_started = 1 + doc = doc $0 "\n" + if ($0 ~ /^[[:space:]]*kind:[[:space:]]*PersistentVolume([[:space:]]|$)/) { + doc_is_pv = 1 + } + } + + END { + flush_doc() + } + ' "$k8s_file" > "$temp_k8s_file" + + if [[ ! -s "$temp_k8s_file" ]]; then + rm -f -- "$k8s_file" + rm -f -- "$temp_k8s_file" + print_info "Removed persistent-volume-only manifest file: $k8s_file" + else + mv -- "$temp_k8s_file" "$k8s_file" + fi + done < <(find "$K8S_DIR" -type f \( -name "*.yaml" -o -name "*.yml" \) -print0) + + print_info "PersistentVolume resource definitions removed from Kubernetes manifests" + else + print_info "No PersistentVolume resource found in Kubernetes manifests" + fi + + # Normalize line endings to LF for kubernetes manifests. + while IFS= read -r -d '' k8s_file; do + sed -i 's/\r$//' "$k8s_file" + done < <(find "$K8S_DIR" -type f \( -name "*.yaml" -o -name "*.yml" \) -print0) + print_info "Normalized line endings to LF for Kubernetes YAML files" +else + print_warning "Directory not found, skipping Kubernetes ingress conversion: $K8S_DIR" +fi + +# Conditional directory restructuring: check if src/ and containers/ exist, +# and if containers/ has exactly 1 subdirectory +print_info "Checking conditions for directory restructuring..." + +SRC_PATH="$DEST_DIR/src" +CONTAINERS_PATH="$DEST_DIR/containers" +RESTRUCTURE=false +CONTAINERS_SUBDIR="" + +if [[ -d "$SRC_PATH" ]]; then + print_info " ✓ Found: $SRC_PATH" + + if [[ -d "$CONTAINERS_PATH" ]]; then + print_info " ✓ Found: $CONTAINERS_PATH" + + # Count subdirectories in containers/ + subdir_count=0 + subdirs=() + + while IFS= read -r subdir; do + subdirs+=("$subdir") + subdir_count=$((subdir_count + 1)) + done < <(find "$CONTAINERS_PATH" -mindepth 1 -maxdepth 1 -type d) + + if [[ $subdir_count -eq 1 ]]; then + CONTAINERS_SUBDIR="${subdirs[0]##*/}" # Extract basename + print_info " ✓ Found exactly 1 subdirectory in containers: $CONTAINERS_SUBDIR" + RESTRUCTURE=true + elif [[ $subdir_count -eq 0 ]]; then + print_warning " ✗ No subdirectories found in $CONTAINERS_PATH" + else + print_warning " ✗ Found $subdir_count subdirectories in $CONTAINERS_PATH (expected 1)" + fi + else + print_warning " ✗ Directory not found: $CONTAINERS_PATH" + fi +else + print_warning " ✗ Directory not found: $SRC_PATH" +fi + +if [[ "$RESTRUCTURE" == true ]]; then + print_info "========== STARTING CONDITIONAL DIRECTORY RESTRUCTURING ==========" + print_info "Target subdirectory name: $CONTAINERS_SUBDIR" + + # 1) Rename src -> _src + if [[ -d "$SRC_PATH" ]]; then + mv -- "$SRC_PATH" "$DEST_DIR/_src" + print_info "Step 1/6: Renamed src -> _src" + fi + + # 2) Create src// + mkdir -p "$DEST_DIR/src/$CONTAINERS_SUBDIR" + print_info "Step 2/6: Created directory src/$CONTAINERS_SUBDIR" + + # 3) Move _src -> src// + mv -- "$DEST_DIR/_src" "$DEST_DIR/src/$CONTAINERS_SUBDIR/_src" + print_info "Step 3/6: Moved _src -> src/$CONTAINERS_SUBDIR/_src" + + # 4) Rename _src -> src (within src//) + mv -- "$DEST_DIR/src/$CONTAINERS_SUBDIR/_src" "$DEST_DIR/src/$CONTAINERS_SUBDIR/src" + print_info "Step 4/6: Renamed _src -> src/$CONTAINERS_SUBDIR/src" + + # 5) Move *.json, *.js, .prettierrc from root to src// + print_info "Step 5/6: Moving files from root to src/$CONTAINERS_SUBDIR/..." + + # Move *.json files + while IFS= read -r json_file; do + if [[ -f "$json_file" ]]; then + mv -- "$json_file" "$DEST_DIR/src/$CONTAINERS_SUBDIR/" + print_info " Moved: $(basename "$json_file") -> src/$CONTAINERS_SUBDIR/" + fi + done < <(find "$DEST_DIR" -maxdepth 1 -type f -name "*.json") + + # Move *.js files + while IFS= read -r js_file; do + if [[ -f "$js_file" ]]; then + mv -- "$js_file" "$DEST_DIR/src/$CONTAINERS_SUBDIR/" + print_info " Moved: $(basename "$js_file") -> src/$CONTAINERS_SUBDIR/" + fi + done < <(find "$DEST_DIR" -maxdepth 1 -type f -name "*.js") + + # Move .prettierrc if exists + if [[ -f "$DEST_DIR/.prettierrc" ]]; then + mv -- "$DEST_DIR/.prettierrc" "$DEST_DIR/src/$CONTAINERS_SUBDIR/" + print_info " Moved: .prettierrc -> src/$CONTAINERS_SUBDIR/" + fi + + # 6) Move envs/ directory if exists + if [[ -d "$DEST_DIR/envs" ]]; then + mv -- "$DEST_DIR/envs" "$DEST_DIR/src/$CONTAINERS_SUBDIR/" + print_info "Step 6/6: Moved envs directory -> src/$CONTAINERS_SUBDIR/envs" + else + print_warning "Step 6/6: envs directory not found at root, skipping move" + fi + + print_info "========== DIRECTORY RESTRUCTURING COMPLETED SUCCESSFULLY ==========" +else + print_info "Directory restructuring conditions not met, skipping directory reorganization" +fi +fi # end TRANSFORMATION_ONLY skip for directory operations + +# Copy transformed content from source repo to target repo +print_info "Copying transformed content: $DEST_DIR -> $GITEA_DEST_DIR ..." +cp -R "./${DEST_DIR}/." "./${GITEA_DEST_DIR}/" +print_info "Copy completed" + +# Commit and push changes on target Gitea repository with skip CI/CD marker +print_info "Preparing commit and push in $GITEA_DEST_DIR ..." +git -C "$GITEA_DEST_DIR" add -A + +if git -C "$GITEA_DEST_DIR" diff --cached --quiet; then + print_warning "No changes detected in $GITEA_DEST_DIR, skipping commit and push" +else + COMMIT_MESSAGE="chore: sync transformed content [skip ci]" + git -C "$GITEA_DEST_DIR" commit -m "$COMMIT_MESSAGE" + git -C "$GITEA_DEST_DIR" push + print_info "Commit and push completed" +fi + +exit 0 diff --git a/st2idc/httproute.yaml b/st2idc/httproute.yaml new file mode 100644 index 0000000..bf2aa6d --- /dev/null +++ b/st2idc/httproute.yaml @@ -0,0 +1,18 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute2 +spec: + hostnames: + - + parentRefs: + - name: main-gateway + namespace: nginx-gateway + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: /project/inbound_figc/journal_old2new.out +30 4 * * * /project/inbound_figc/mainload.sh > /project/inbound_figc/loadmain.out +0 3 * * * /project/inbound_figc/backup.sh > /project/inbound_figc/backup.out \ No newline at end of file diff --git a/st2idc/lnd-script/gareload.sql b/st2idc/lnd-script/gareload.sql new file mode 100644 index 0000000..442d1a7 --- /dev/null +++ b/st2idc/lnd-script/gareload.sql @@ -0,0 +1 @@ +SELECT "aaagare"(); \ No newline at end of file diff --git a/st2idc/lnd-script/getftp.sh b/st2idc/lnd-script/getftp.sh new file mode 100644 index 0000000..b4e1a35 --- /dev/null +++ b/st2idc/lnd-script/getftp.sh @@ -0,0 +1,11 @@ +userid="sportftp" +password="\$porteams2019" + + +ftp -n 212.77.67.140 << EOFTP >ftpget.$$ +user $userid $password +prompt +cd files +mget *.csv +quit +EOFTP \ No newline at end of file diff --git a/st2idc/lnd-script/journal.sql b/st2idc/lnd-script/journal.sql new file mode 100644 index 0000000..9678cd3 --- /dev/null +++ b/st2idc/lnd-script/journal.sql @@ -0,0 +1,14 @@ +delete from journal_ftp; +insert into journal_ftp(id,chi,quando,idp,ida,ora,data,idc) select * from dblink('dbname=pgare','select id,chi,quando,idp,ida,ora,data,idc from vjournal') AS journal(id integer,chi numeric,quando timestamp,idp integer,ida integer,ora text,data date,idc varchar); +insert into journal_figc(id_utente_modificatore,matricola_figc_sq_modificatrice,created_at,id_partita,id_azione,orario,data,matricola_impianto_figc,azione_annullata,id_old) +select +null as id_utente_modificatore, +null as matricola_figc_sq_modificatrice, +j.quando as created_at, +p.id as id_partita, +j.ida as id_azione, +j.ora::time as orario, +j.data as data, +idc as matricola_impianto_figc, +false as azione_annullata,j.id +from journal_ftp j,partite_figc p where j.idp=p.id_partita_ftp and j.id not in(select id_old from journal_figc) order by j.id; \ No newline at end of file diff --git a/st2idc/lnd-script/journal2.sql b/st2idc/lnd-script/journal2.sql new file mode 100644 index 0000000..c85d0ae --- /dev/null +++ b/st2idc/lnd-script/journal2.sql @@ -0,0 +1,12 @@ +insert into journal_figc(id_utente_modificatore,matricola_figc_sq_modificatrice,created_at,id_partita,id_azione,orario,data,matricola_impianto_figc,azione_annullata,id_old) +select +null as id_utente_modificatore, +null as matricola_figc_sq_modificatrice, +j.quando as created_at, +p.id as id_partita, +j.ida as id_azione, +j.ora::time as orario, +j.data as data, +idc as matricola_impianto_figc, +false as azione_annullata,j.id +from journal_ftp j,partite_figc p where j.idp=p.id_partita_ftp and j.id not in(select id_old from journal_figc) order by j.id; \ No newline at end of file diff --git a/st2idc/lnd-script/journal_old2new.sh b/st2idc/lnd-script/journal_old2new.sh new file mode 100644 index 0000000..193f6d4 --- /dev/null +++ b/st2idc/lnd-script/journal_old2new.sh @@ -0,0 +1,5 @@ +#!/bin/bash +. ./.profile +cd /project/inbound_figc +/usr/local/bin/kubectl cp journal.sql db/postgresql-1-postgresql-0:/etc/journal.sql -c postgresql-server +/usr/local/bin/kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n db -- psql -U postgres -d gare -f /etc/journal.sql \ No newline at end of file diff --git a/st2idc/lnd-script/loadclub.sql b/st2idc/lnd-script/loadclub.sql new file mode 100644 index 0000000..fbc0fee --- /dev/null +++ b/st2idc/lnd-script/loadclub.sql @@ -0,0 +1 @@ +insert into partite(idpf,ids) select distinct * from pgpartite2insert; \ No newline at end of file diff --git a/st2idc/lnd-script/loadpgare.sh b/st2idc/lnd-script/loadpgare.sh new file mode 100644 index 0000000..2902a59 --- /dev/null +++ b/st2idc/lnd-script/loadpgare.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +sed -i "s/\"/ /g" $1 + +if [[ $2 == "cal" ]] +then +echo "strutr;r0cmp;r0gir;r0gio;r0anr;r0dai;r0ma1;r0ma2;r0no1;r0pu1;r0pu2;void1;void2;void3;void4;void5;void6;void7;r0imp;r0ori;void8"|cat - $1 > /tmp/out && mv /tmp/out $1"_int" +fi + + +if [[ $2 == "imp" ]] +then +echo "i0str;i0imp;i0dei;i0ini;i0loi;i0prv;void"|cat - $1 > /tmp/out && mv /tmp/out $1"_int" +fi + +if [[ $2 == "ana" ]] +then +echo "SCOSO;SCOMO;SCOMI;SCOMC;SCOMF;SSIGL;SDENO;DESCA;X21;X23;X24;SPC01;SIND1;SIND2;SCAPS;SLOCR;SPVRE;SINC1;SINC2;SCAPC;SLOCO;SPVCO;STEDS;STEPR;STESR;SCAPO;SFAXS;STELX;SPIVA;SNATU;SSITO;SMAIL;SPRESC;SPRESN;F0DEI;F0INI;F0LOI;F9COI;F0PRV;F0CAP;C0DEI;C0INI;C0LOI;C9COI;C0PRV;C0CAP;D0DEI;D0INI;D0LOI;D9COI;D0PRV;D0CAP;SAT01;SAT02;SAT03;SAT04;SAT05;SAT06;SAT07;SAT08;SAT09;SAT10;void"|cat - $1 > /tmp/out && mv /tmp/out $1"_int" +fi + +if [[ $2 == "cla" ]] +then +echo "REG;KSGTCL;GIRONE;IDSOC;PUNTI;GF;GS;GIOCATE;VINTE;PAREGGIATE;perse"|cat - $1 > /tmp/out && mv /tmp/out $1"_int" +fi + + + +iconv -f iso-8859-1 -t UTF-8 $1"_int" -o $1"_int_utf8" +sed '1s/^\xEF\xBB\xBF//' < $1"_int_utf8" >$1"_int_utf8_nobom.csv" +sed -i '1s/^\xEF\xBB\xBF//' $1"_int_utf8" \ No newline at end of file diff --git a/st2idc/lnd-script/mainload.sh b/st2idc/lnd-script/mainload.sh new file mode 100644 index 0000000..e69de29 diff --git a/st2idc/lnd-script/mainload_dev.sh b/st2idc/lnd-script/mainload_dev.sh new file mode 100644 index 0000000..c8c7e26 --- /dev/null +++ b/st2idc/lnd-script/mainload_dev.sh @@ -0,0 +1,15 @@ +#!/bin/bash +cd /project/inbound_figc + +kubectl cp cserc17f.csv_int_utf8_nobom.csv dbdev/postgresql-1-postgresql-0:/etc/cserc17f_2load_utf8_nobom.csv +kubectl cp cseri17f.csv_int_utf8_nobom.csv dbdev/postgresql-1-postgresql-0:/etc/cseri17f_2load_utf8_nobom.csv +kubectl cp csexc17f.csv_int_utf8_nobom.csv dbdev/postgresql-1-postgresql-0:/etc/csexc17f_2load_utf8_nobom.csv +kubectl cp csezc17f.csv_int_utf8_nobom.csv dbdev/postgresql-1-postgresql-0:/etc/csezc17f_2load_utf8_nobom.csv +#k cp csezc17f.csv_int_utf8_nobom.csv dbdev/postgresql-1-postgresql-0:/etc/csezc17f_2load_utf8_nobom.csv + + +kubectl cp gareload.sql dbdev/postgresql-1-postgresql-0:/etc/gareload.sql +kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n dbdev -- psql -U postgres -d gare -f /etc/gareload.sql + +#kubectl cp loadclub.sql db/postgresql-1-postgresql-0:/etc/loadclub.sql +#kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n db -- psql -U postgres -d segdigi -f /etc/loadclub.sql \ No newline at end of file diff --git a/st2idc/lnd-script/partite_ftp.sql b/st2idc/lnd-script/partite_ftp.sql new file mode 100644 index 0000000..a7d9956 --- /dev/null +++ b/st2idc/lnd-script/partite_ftp.sql @@ -0,0 +1,2 @@ +delete from partite_ftp; +insert into partite_ftp(id,strutr,r0cmp,r0gir,r0gio,r0anr,r0dai,r0ma1,r0ma2,r0pu1,r0pu2,r0cov,r0rdi,rfl0,r0imp,r0ori,color,data,valid) select * from dblink('dbname=pgare','select id,strutr,r0cmp,r0gir,r0gio,r0anr,r0dai,r0ma1,r0ma2,r0pu1,r0pu2,r0cov,r0rdi,r0fl0,r0imp,r0ori,color,data,valid from calendari') AS calendari(id integer, strutr integer,r0cmp character(8),r0gir character(8),r0gio character(8),r0anr character(3),r0dai character(6),r0ma1 integer,r0ma2 integer,r0pu1 character(3),r0pu2 character(3),r0cov character(3),r0rdi character(3),r0fl0 character(3),r0imp character(8),r0ori character(4),color character(10),data date,valid boolean); \ No newline at end of file diff --git a/st2idc/lnd-script/pgareload.sql b/st2idc/lnd-script/pgareload.sql new file mode 100644 index 0000000..e4ddd38 --- /dev/null +++ b/st2idc/lnd-script/pgareload.sql @@ -0,0 +1,4 @@ +SELECT "aaapgare"(); +SELECT "aaapgare_imp"(); +SELECT "aaapgare_ana"(); +SELECT "aaacla"(); \ No newline at end of file diff --git a/st2idc/restorePG.sh b/st2idc/restorePG.sh new file mode 100644 index 0000000..6d1254d --- /dev/null +++ b/st2idc/restorePG.sh @@ -0,0 +1,162 @@ +#!/bin/bash +set -Eeuo pipefail + +# Configuration +TARGET_ENV="${1:-stsharedservices-dev}" +DBNAME="${2:-geco}" +POD_NAME="${3:-stpgclusterdev-1}" +CONTAINER_NAME="${4:-postgres}" +BACKUP_FILE="${5:-.}" +DROP_IF_EXISTS="${6:-true}" +DB_USER="${7:-${PGUSER:-postgres}}" +DB_PASSWORD="${8:-${PGPASSWORD:-}}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Logging functions +log() { + echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $*" +} + +error() { + echo -e "${RED}[$(date '+%Y-%m-%d %H:%M:%S')] ERROR:${NC} $*" >&2 + exit 1 +} + +warn() { + echo -e "${YELLOW}[$(date '+%Y-%m-%d %H:%M:%S')] WARN:${NC} $*" +} + +info() { + echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] INFO:${NC} $*" +} + +log "Inizio restore database PostgreSQL" +log "Namespace target: ${TARGET_ENV}" +log "Pod target: ${POD_NAME}" +log "Database: ${DBNAME}" +log "DB user: ${DB_USER}" +log "File backup: ${BACKUP_FILE}" + +# Validate backup file exists +if [ ! -f "${BACKUP_FILE}" ]; then + error "File di backup non trovato: ${BACKUP_FILE}" +fi + +FILE_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1) +log "Dimensione file backup: ${FILE_SIZE}" + +# Determine if backup is compressed +if [[ "${BACKUP_FILE}" == *.gz ]]; then + IS_COMPRESSED=true + info "Backup compresso (gzip) rilevato" +else + IS_COMPRESSED=false + info "Backup non compresso rilevato" +fi + +# Check if namespace exists +if ! kubectl get namespace "${TARGET_ENV}" &>/dev/null; then + error "Namespace '${TARGET_ENV}' non trovato" +fi + +# Check if pod exists +if ! kubectl get pod "${POD_NAME}" -n "${TARGET_ENV}" &>/dev/null; then + error "Pod '${POD_NAME}' non trovato nel namespace '${TARGET_ENV}'" +fi + +# Check if pod is running +POD_STATUS=$(kubectl get pod "${POD_NAME}" -n "${TARGET_ENV}" -o jsonpath='{.status.phase}') +if [ "${POD_STATUS}" != "Running" ]; then + error "Pod '${POD_NAME}' non è in stato Running (stato attuale: ${POD_STATUS})" +fi + +# Check database connectivity +info "Verifica connessione al database..." +if ! kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" pg_isready -U "${DB_USER}" -d "${DBNAME}" &>/dev/null; then + error "Impossibile connettersi a PostgreSQL nel pod '${POD_NAME}'" +fi + +log "Connessione al database verificata" + +# Drop database if requested +if [ "${DROP_IF_EXISTS}" = "true" ]; then + warn "Elimino il database '${DBNAME}' se esiste..." + kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -c "DROP DATABASE IF EXISTS \"${DBNAME}\" WITH (FORCE);" 2>/dev/null || warn "Database non esiste o elimina non è riuscita" + + info "Creo nuovo database '${DBNAME}'..." + kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -c "CREATE DATABASE \"${DBNAME}\";" || error "Impossibile creare il database '${DBNAME}'" +fi + +log "Inizio restore del database..." + +# Execute restore +if [ "${IS_COMPRESSED}" = "true" ]; then + info "Decompressione e restore in corso..." + gunzip -c "${BACKUP_FILE}" | kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" pg_restore -U "${DB_USER}" -d "${DBNAME}" --no-owner -v 2>&1 | grep -E "(NOTICE|ERROR|completed)" || true +else + info "Restore da file plain SQL in corso..." + cat "${BACKUP_FILE}" | kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -d "${DBNAME}" -v ON_ERROR_STOP=1 2>&1 | grep -E "(NOTICE|ERROR|INSERT|CREATE)" | tail -20 || true +fi + +log "Restore completato" + +info "Allineo owner delle tabelle al proprietario del database target..." +TARGET_DB_OWNER=$(kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -d postgres -t -A -c "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname='${DBNAME}';") + +if [ -z "${TARGET_DB_OWNER}" ]; then + error "Impossibile determinare l'owner del database '${DBNAME}'" +fi + +kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -d "${DBNAME}" -v ON_ERROR_STOP=1 -c "DO \$\$ DECLARE r RECORD; BEGIN FOR r IN SELECT n.nspname, c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r', 'p') AND n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg_toast%' LOOP EXECUTE format('ALTER TABLE %I.%I OWNER TO %I', r.nspname, r.relname, '${TARGET_DB_OWNER}'); END LOOP; END \$\$;" >/dev/null + +log "Owner tabelle allineato a '${TARGET_DB_OWNER}'" + +# Verify restore success +info "Verifica integrità database..." +TABLE_COUNT=$(kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -d "${DBNAME}" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public';" 2>/dev/null || echo "0") + +if [ "${TABLE_COUNT}" -gt 0 ]; then + log "✅ Restore completato con successo" + log "Database '${DBNAME}' contiene ${TABLE_COUNT} tabelle" +else + warn "⚠️ Database '${DBNAME}' non contiene tabelle pubbliche (potrebbe essere empty)" +fi + +# Get database size +DB_SIZE=$(kubectl exec -i "${POD_NAME}" \ + -c "${CONTAINER_NAME}" \ + -n "${TARGET_ENV}" \ + -- env "PGPASSWORD=${DB_PASSWORD}" psql -U "${DB_USER}" -t -c "SELECT pg_size_pretty(pg_database_size('${DBNAME}'));" 2>/dev/null || echo "N/A") + +log "Dimensione database: ${DB_SIZE}" +log "✅ Restore completato con successo" diff --git a/st2idc/run-clone-and-transform.sh b/st2idc/run-clone-and-transform.sh new file mode 100644 index 0000000..1be6c4b --- /dev/null +++ b/st2idc/run-clone-and-transform.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +# Launcher script for clone-and-transform.sh +# Fill all values below before running. + + +GITLAB_URL="https://gitlab.com" +GITLAB_TOKEN="glpat-E67BjdXoKvSEgHG31XZW0GM6MQpvOjEKdTozNzF5OQ8.01.171191k74" +GITLAB_REPO_PATH=$1 +DEST_DIR="./work-repo" +TRANSFORM_FILE=$3 +GITEA_REPO_PATH=$2 +TRANSFORMATION_ONLY="${4:-false}" +# +#GITLAB_URL="https://gitlab.com" +#GITLAB_TOKEN="glpat-xxxxxxxxxxxxxxxxxxxx" +#GITLAB_REPO_PATH="group/project" +#DEST_DIR="./work-repo" +#TRANSFORM_FILE="./transform.env" +#GITEA_REPO_PATH="group/project" + +# Set to "true" to skip GitLab clone and run only: Gitea clone + transform + commit/push + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MAIN_SCRIPT="${SCRIPT_DIR}/clone-and-transform.sh" + +if [[ ! -f "$MAIN_SCRIPT" ]]; then + echo "[ERROR] Main script not found: $MAIN_SCRIPT" + exit 1 +fi + +chmod +x "$MAIN_SCRIPT" + +if [[ "$TRANSFORMATION_ONLY" == "true" ]]; then + "$MAIN_SCRIPT" \ + -transformation \ + --dest "$DEST_DIR" \ + --transform "$TRANSFORM_FILE" \ + --gitea-repo "$GITEA_REPO_PATH" +else + "$MAIN_SCRIPT" \ + --url "$GITLAB_URL" \ + --token "$GITLAB_TOKEN" \ + --repo "$GITLAB_REPO_PATH" \ + --dest "$DEST_DIR" \ + --transform "$TRANSFORM_FILE" \ + --gitea-repo "$GITEA_REPO_PATH" +fi diff --git a/st2idc/st2idc.txt b/st2idc/st2idc.txt new file mode 100644 index 0000000..36b49de --- /dev/null +++ b/st2idc/st2idc.txt @@ -0,0 +1,49 @@ +---1) creazione nuova org su gitea Sporteams +---2) crezione progetto stsharedservices +---3) deploy postregsdb su stsharedservices +---4) deploy rabbitmq su stsharedservices +---5) export db Geco +---6) import db Geco +---7) creazione progetti gecofe e gecobe +8) clonazione da analoghi st +9) deploy + + + + + +kubectl cnpg hibernate on pg-athleteos -n athleteos-dev + + +stpgclusterdev-rw.stsharedservices-dev.svc.cluster.local + +kubectl cnpg delete cluster stpgclusterdev + +devVkn9tcHSxK7E + + +Backup +#!/bin/sh +source_env=dbdev +#target_env=dbqa +dbname=geco + +backupfile=${dbname}.backup +#cd /shared/dbdata/sql + +kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n ${source_env} -- pg_dump -U postgres -d ${dbname} -F c -b -v -f /shared/backup/$backupfile +kubectl cp ${source_env}/postgresql-1-postgresql-0:/shared/backup/${backupfile} ${backupfile} + +#mv $backupfile restore_${backupfile} +kubectl cp ${backupfile} ${target_env}/postgresql-1-postgresql-0:/tmp/restore_${backupfile} +kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n ${target_env} -- psql -U postgres -c "drop database ${dbname}" +kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n ${target_env} -- psql -U postgres -c "create database ${dbname}" +kubectl exec -it postgresql-1-postgresql-0 -c postgresql-server -n ${target_env} -- pg_restore -U postgres -d ${dbname} /tmp/restore_${backupfile} + + +scp geco-20260418-171917.backup.gz barucci@10.20.1.101:/root/work/fromst + +iZmBf42AVzDtD8ZcA9pFvtIGX6cbg9VvzjE0Jd7LkZnplvpceCAYjLqQRr7Z7SsA + + +rabbitmqcluster.dbdev.svc.cluster.local=stmqclusterdev.stsharedservices-dev.svc.cluster.local \ No newline at end of file diff --git a/st_migrationlist.xlsx b/st2idc/st_migrationlist.xlsx similarity index 100% rename from st_migrationlist.xlsx rename to st2idc/st_migrationlist.xlsx diff --git a/st_size.csv b/st2idc/st_size.csv similarity index 100% rename from st_size.csv rename to st2idc/st_size.csv diff --git a/st_size.xlsx b/st2idc/st_size.xlsx similarity index 100% rename from st_size.xlsx rename to st2idc/st_size.xlsx diff --git a/st2idc/transform.env b/st2idc/transform.env new file mode 100644 index 0000000..b41b3bf --- /dev/null +++ b/st2idc/transform.env @@ -0,0 +1,33 @@ +# Example transformations: +# DATABASE_HOST=localhost=DATABASE_HOST=10.0.0.5 +# API_PORT=8080=API_PORT=3000 +# ENVIRONMENT=dev=ENVIRONMENT=prod +# OLD_SECRET_KEY=abc123=NEW_SECRET_KEY=xyz789 + +#Postgres uri +postgresql-1-postgresql-svc.dbdev.svc.cluster.local=postgresql-rw.stsharedservices-dev.svc.cluster.local + +#dns domain +sporteams.app=pigreco66.it + +#postgres user password +devVkn9tcHSxK7E=iZmBf42AVzDtD8ZcA9pFvtIGX6cbg9VvzjE0Jd7LkZnplvpceCAYjLqQRr7Z7SsA + +#rabbitMQ uri +rabbitmqcluster.dbdev.svc.cluster.local=stmqclusterdev.stsharedservices-dev.svc.cluster.local + +#RABBITMQ_USER +t2rW7tOSXkq2Oh6XZIrVlBp8Cd3kzd7M=default_user_wN2410ODWhkuXJynz9C + +#RABBITMQ_PASSWORD +q2zB-eeyJwDltXDhEFzJ7QhPsrOg4XDe=oENtKdDBUWOePk5OTQ36JInQ8XPsZD8W + +#GECO_INTERNAL_URL = +nest.gecobedev=nest.st-geco-backend + +#docker image +eu.gcr.io/sporteamscloud/nest%name%:%tag%= +eu.gcr.io/sporteamscloud/nginx%name%:%tag%= + +#parametri +%backendurl%= \ No newline at end of file