Fix: Update CI workflow to use Alpine-based images, install Node.js and Trivy with improved methods, and enhance dependency scanning steps #1

Merged
ilia merged 11 commits from update-ci into master 2025-12-17 22:45:01 -05:00
18 changed files with 275 additions and 154 deletions

View File

@ -4,11 +4,14 @@
exclude_paths:
- .cache/
- .github/
- .gitea/
- .ansible/
# Skip specific rules
skip_list:
- yaml[line-length] # Allow longer lines in some cases
- yaml[document-start] # Allow missing document start in vault files
- yaml[truthy] # Allow different truthy values in workflow files
- name[casing] # Allow mixed case in task names
- args[module] # Skip args rule that causes "file name too long" issues
- var-naming[no-role-prefix] # Allow shorter variable names for readability

View File

@ -32,7 +32,7 @@ jobs:
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl
apt-get update && apt-get install -y curl git
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
@ -62,109 +62,35 @@ jobs:
secret-scanning:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
image: zricethezav/gitleaks:latest
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
apk add --no-cache nodejs npm curl
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install git and Gitleaks
run: |
apt-get update && apt-get install -y wget curl git
GITLEAKS_VERSION=$(curl -s https://api.github.com/repos/gitleaks/gitleaks/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
wget -q "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" -O /tmp/gitleaks.tar.gz
tar -xzf /tmp/gitleaks.tar.gz -C /usr/local/bin/ gitleaks
chmod +x /usr/local/bin/gitleaks
gitleaks version
- name: Run Gitleaks secret scan
run: |
gitleaks detect --source . --verbose --no-banner --exit-code 1
- name: Scan for secrets
run: gitleaks detect --source . --no-banner --redact --exit-code 0
continue-on-error: true
dependency-scan:
runs-on: ubuntu-latest
container:
image: ubuntu:22.04
image: aquasec/trivy:latest
steps:
- name: Install Node.js for checkout action
run: |
apt-get update && apt-get install -y curl
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
apt-get install -y nodejs
apk add --no-cache nodejs npm curl
- name: Check out code
uses: actions/checkout@v4
- name: Install Trivy
run: |
apt-get update && apt-get install -y wget curl tar
# Try multiple download methods for reliability
echo "Downloading Trivy..."
if wget -q "https://github.com/aquasecurity/trivy/releases/latest/download/trivy_linux_amd64.tar.gz" -O /tmp/trivy.tar.gz 2>&1; then
echo "Downloaded tar.gz, extracting..."
tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy
mv /tmp/trivy /usr/local/bin/trivy
elif wget -q "https://github.com/aquasecurity/trivy/releases/latest/download/trivy_linux_amd64" -O /usr/local/bin/trivy 2>&1; then
echo "Downloaded binary directly"
else
echo "Failed to download Trivy, trying with version detection..."
TRIVY_VERSION=$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
wget -q "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -O /tmp/trivy.tar.gz
tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy
mv /tmp/trivy /usr/local/bin/trivy
fi
chmod +x /usr/local/bin/trivy
/usr/local/bin/trivy --version
trivy --version
- name: Scan npm dependencies
run: |
if [ -f "package.json" ]; then
echo "Scanning npm dependencies..."
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table --exit-code 0 .
else
echo "No package.json found, skipping npm scan"
fi
continue-on-error: true
- name: Scan Python dependencies
run: |
if [ -f "requirements.txt" ]; then
echo "Scanning Python dependencies..."
trivy fs --scanners vuln --severity HIGH,CRITICAL --format table --exit-code 0 .
else
echo "No requirements.txt found, skipping Python scan"
fi
continue-on-error: true
- name: Generate dependency scan report
run: |
echo "Generating comprehensive scan report..."
trivy fs --scanners vuln --format json --output trivy-report.json . || true
trivy fs --scanners vuln --format table . || true
- name: Display Trivy report summary
if: always()
run: |
echo "## Trivy Dependency Scan Results" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
if [ -f trivy-report.json ]; then
echo "✅ Trivy report generated successfully" >> $GITHUB_STEP_SUMMARY || true
echo "📄 Report location: trivy-report.json" >> $GITHUB_STEP_SUMMARY || true
echo "" >> $GITHUB_STEP_SUMMARY || true
echo "Note: Artifact upload not available in Gitea Actions" >> $GITHUB_STEP_SUMMARY || true
echo "Report details are available in the job logs above." >> $GITHUB_STEP_SUMMARY || true
else
echo "⚠️ Trivy report file not found" >> $GITHUB_STEP_SUMMARY || true
fi
continue-on-error: true
- name: Scan dependencies
run: trivy fs --scanners vuln,secret --exit-code 0 .
sast-scan:
runs-on: ubuntu-latest
@ -292,8 +218,11 @@ jobs:
fi
done
if [ $failed -eq 1 ]; then
echo "Some playbooks have errors (this is expected without inventory/vault)"
exit 0
echo "❌ Some playbooks have syntax errors!"
echo "Note: This may be expected if playbooks require inventory/vault, but syntax errors should still be fixed."
exit 1
else
echo "✅ All playbooks passed syntax check"
fi
continue-on-error: true
@ -313,22 +242,43 @@ jobs:
- name: Install Trivy
run: |
set -e
apt-get update && apt-get install -y wget curl tar
# Try multiple download methods for reliability
echo "Downloading Trivy..."
if wget -q "https://github.com/aquasecurity/trivy/releases/latest/download/trivy_linux_amd64.tar.gz" -O /tmp/trivy.tar.gz 2>&1; then
echo "Downloaded tar.gz, extracting..."
tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy
mv /tmp/trivy /usr/local/bin/trivy
elif wget -q "https://github.com/aquasecurity/trivy/releases/latest/download/trivy_linux_amd64" -O /usr/local/bin/trivy 2>&1; then
echo "Downloaded binary directly"
else
echo "Failed to download Trivy, trying with version detection..."
TRIVY_VERSION=$(curl -s https://api.github.com/repos/aquasecurity/trivy/releases/latest | grep tag_name | cut -d '"' -f 4 | sed 's/v//')
wget -q "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -O /tmp/trivy.tar.gz
tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy
mv /tmp/trivy /usr/local/bin/trivy
# Use a fixed, known-good Trivy version to avoid URL/redirect issues
TRIVY_VERSION="0.58.2"
TRIVY_URL="https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz"
echo "Installing Trivy version: ${TRIVY_VERSION}"
echo "Downloading from: ${TRIVY_URL}"
if ! wget --progress=bar:force "${TRIVY_URL}" -O /tmp/trivy.tar.gz 2>&1; then
echo "❌ Failed to download Trivy archive"
echo "Checking if file was partially downloaded:"
ls -lh /tmp/trivy.tar.gz 2>/dev/null || echo "No file found"
exit 1
fi
if [ ! -f /tmp/trivy.tar.gz ] || [ ! -s /tmp/trivy.tar.gz ]; then
echo "❌ Downloaded Trivy archive is missing or empty"
exit 1
fi
echo "Download complete. File size: $(du -h /tmp/trivy.tar.gz | cut -f1)"
echo "Extracting Trivy..."
if ! tar -xzf /tmp/trivy.tar.gz -C /tmp/ trivy; then
echo "❌ Failed to extract Trivy binary from archive"
tar -tzf /tmp/trivy.tar.gz 2>&1 | head -20 || true
exit 1
fi
if [ ! -f /tmp/trivy ]; then
echo "❌ Trivy binary not found after extraction"
ls -la /tmp/ | grep trivy || ls -la /tmp/ | head -20
exit 1
fi
mv /tmp/trivy /usr/local/bin/trivy
chmod +x /usr/local/bin/trivy
/usr/local/bin/trivy --version
trivy --version
@ -349,7 +299,7 @@ jobs:
sonar-analysis:
runs-on: ubuntu-latest
container:
image: sonarsource/sonar-scanner-cli:latest
image: ubuntu:22.04
env:
SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@ -363,13 +313,127 @@ jobs:
- name: Check out code
uses: actions/checkout@v4
- name: Install Java and SonarScanner
run: |
set -e
apt-get update && apt-get install -y wget curl unzip openjdk-21-jre
# Use a known working version to avoid download issues
SONAR_SCANNER_VERSION="5.0.1.3006"
SCANNER_URL="https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip"
echo "Installing SonarScanner version: ${SONAR_SCANNER_VERSION}"
echo "Downloading from: ${SCANNER_URL}"
# Download with verbose error output
if ! wget --progress=bar:force "${SCANNER_URL}" -O /tmp/sonar-scanner.zip 2>&1; then
echo "❌ Failed to download SonarScanner"
echo "Checking if file was partially downloaded:"
ls -lh /tmp/sonar-scanner.zip 2>/dev/null || echo "No file found"
exit 1
fi
# Verify download
if [ ! -f /tmp/sonar-scanner.zip ] || [ ! -s /tmp/sonar-scanner.zip ]; then
echo "❌ Downloaded file is missing or empty"
exit 1
fi
echo "Download complete. File size: $(du -h /tmp/sonar-scanner.zip | cut -f1)"
echo "Extracting SonarScanner..."
if ! unzip -q /tmp/sonar-scanner.zip -d /tmp; then
echo "❌ Failed to extract SonarScanner"
echo "Archive info:"
file /tmp/sonar-scanner.zip || true
unzip -l /tmp/sonar-scanner.zip 2>&1 | head -20 || true
exit 1
fi
# Find the extracted directory (handle both naming conventions)
EXTRACTED_DIR=""
if [ -d "/tmp/sonar-scanner-${SONAR_SCANNER_VERSION}-linux" ]; then
EXTRACTED_DIR="/tmp/sonar-scanner-${SONAR_SCANNER_VERSION}-linux"
elif [ -d "/tmp/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux" ]; then
EXTRACTED_DIR="/tmp/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux"
else
# Try to find any sonar-scanner directory
EXTRACTED_DIR=$(find /tmp -maxdepth 1 -type d -name "*sonar-scanner*" | head -1)
fi
if [ -z "$EXTRACTED_DIR" ] || [ ! -d "$EXTRACTED_DIR" ]; then
echo "❌ SonarScanner directory not found after extraction"
echo "Contents of /tmp:"
ls -la /tmp/ | grep -E "(sonar|zip)" || ls -la /tmp/ | head -20
exit 1
fi
echo "Found extracted directory: ${EXTRACTED_DIR}"
mv "${EXTRACTED_DIR}" /opt/sonar-scanner
# Create symlink
if [ -f /opt/sonar-scanner/bin/sonar-scanner ]; then
ln -sf /opt/sonar-scanner/bin/sonar-scanner /usr/local/bin/sonar-scanner
chmod +x /opt/sonar-scanner/bin/sonar-scanner
chmod +x /usr/local/bin/sonar-scanner
else
echo "❌ sonar-scanner binary not found in /opt/sonar-scanner/bin/"
echo "Contents of /opt/sonar-scanner/bin/:"
ls -la /opt/sonar-scanner/bin/ || true
exit 1
fi
echo "Verifying installation..."
if ! sonar-scanner --version; then
echo "❌ SonarScanner verification failed"
echo "PATH: $PATH"
which sonar-scanner || echo "sonar-scanner not in PATH"
exit 1
fi
echo "✓ SonarScanner installed successfully"
- name: Verify SonarQube connection
run: |
echo "Checking SonarQube connectivity..."
if [ -z "$SONAR_HOST_URL" ] || [ -z "$SONAR_TOKEN" ]; then
echo "❌ ERROR: SONAR_HOST_URL or SONAR_TOKEN secrets are not set!"
echo "Please configure them in: Repository Settings → Actions → Secrets"
exit 1
fi
echo "✓ Secrets are configured"
echo "SonarQube URL: ${SONAR_HOST_URL}"
echo "Testing connectivity to SonarQube server..."
if curl -f -s -o /dev/null -w "%{http_code}" "${SONAR_HOST_URL}/api/system/status" | grep -q "200"; then
echo "✓ SonarQube server is reachable"
else
echo "⚠️ Warning: Could not verify SonarQube server connectivity"
fi
- name: Run SonarScanner
run: |
sonar-scanner \
-Dsonar.projectKey=ansible-infra \
echo "Starting SonarQube analysis..."
if ! sonar-scanner \
-Dsonar.projectKey=ansible \
-Dsonar.sources=. \
-Dsonar.host.url=${SONAR_HOST_URL} \
-Dsonar.login=${SONAR_TOKEN}
-Dsonar.token=${SONAR_TOKEN} \
-Dsonar.scm.disabled=true \
-Dsonar.python.version=3.10 \
-X; then
echo ""
echo "❌ SonarScanner analysis failed!"
echo ""
echo "Common issues:"
echo " 1. Project 'ansible' doesn't exist in SonarQube"
echo " → Create it manually in SonarQube UI"
echo " 2. Token doesn't have permission to analyze/create project"
echo " → Ensure token has 'Execute Analysis' permission"
echo " 3. Token doesn't have 'Create Projects' permission (if project doesn't exist)"
echo " → Grant this permission in SonarQube user settings"
echo ""
echo "Check SonarQube logs for more details."
exit 1
fi
continue-on-error: true
workflow-summary:

View File

@ -1,3 +1,4 @@
---
ansible_become_password: root
ansible_python_interpreter: /usr/bin/python3

View File

@ -1,3 +1,4 @@
---
# Configure sudo path for git-ci-01
# Sudo may not be in PATH for non-interactive shells
ansible_become_exe: /usr/bin/sudo
@ -5,4 +6,3 @@ ansible_become_method: sudo
# Alternative: if sudo is in a different location, update this
# ansible_become_exe: /usr/local/bin/sudo

View File

@ -7,4 +7,3 @@ ansible_become_method: sudo
# Configure shell for ladmin user
shell_users:
- ladmin

View File

@ -29,6 +29,7 @@
fi
register: brave_key_check
failed_when: false
changed_when: false
when: applications_brave_needs_install
- name: Check if Brave repository exists and is correct
@ -44,6 +45,7 @@
fi
register: brave_repo_check
failed_when: false
changed_when: false
when: applications_brave_needs_install
- name: Clean up duplicate Brave repository files

View File

@ -17,4 +17,3 @@ r_packages:
- r-base
- r-base-dev
- r-recommended

View File

@ -5,4 +5,3 @@
state: restarted
daemon_reload: true
become: true

View File

@ -1,4 +1,3 @@
---
dependencies:
- role: base

View File

@ -200,4 +200,3 @@
- name: Display R version
ansible.builtin.debug:
msg: "R version installed: {{ r_version.stdout_lines[0] if r_version.stdout_lines | length > 0 else 'Not checked in dry-run mode' }}"

View File

@ -30,6 +30,7 @@
fi
register: nodesource_repo_check
failed_when: false
changed_when: false
when: node_version_check.rc != 0 or not node_version_check.stdout.startswith('v22')
- name: Check if NodeSource GPG key exists and is correct
@ -45,6 +46,7 @@
fi
register: nodesource_key_check
failed_when: false
changed_when: false
when: node_version_check.rc != 0 or not node_version_check.stdout.startswith('v22')
- name: Remove incorrect NodeSource repository

View File

@ -12,6 +12,7 @@
fi
register: docker_key_check
failed_when: false
changed_when: false
- name: Remove incorrect Docker GPG key
ansible.builtin.file:
@ -43,4 +44,3 @@
path: /tmp/docker.gpg
state: absent
when: docker_key_check.stdout in ["not_exists", "wrong_key"]

View File

@ -12,6 +12,7 @@
fi
register: docker_repo_check
failed_when: false
changed_when: false
- name: Remove incorrect Docker repository
ansible.builtin.file:
@ -26,4 +27,3 @@
state: present
update_cache: true
when: docker_repo_check.stdout in ["not_exists", "wrong_config"]

View File

@ -20,6 +20,7 @@
fi
register: docker_repo_check
failed_when: false
changed_when: false
- name: Remove incorrect Docker repository
ansible.builtin.file:
@ -34,4 +35,3 @@
state: present
update_cache: true
when: docker_repo_check.stdout in ["not_exists", "wrong_config"]

View File

@ -12,6 +12,7 @@
fi
register: docker_repo_check
failed_when: false
changed_when: false
- name: Remove incorrect Docker repository
ansible.builtin.file:
@ -26,4 +27,3 @@
state: present
update_cache: true
when: docker_repo_check.stdout in ["not_exists", "wrong_config"]

View File

@ -18,6 +18,7 @@
fi
register: tailscale_key_check
failed_when: false
changed_when: false
when: tailscale_version_check.rc != 0
- name: Check if Tailscale repository exists and is correct
@ -33,6 +34,7 @@
fi
register: tailscale_repo_check
failed_when: false
changed_when: false
when: tailscale_version_check.rc != 0
- name: Remove incorrect Tailscale GPG key

View File

@ -141,39 +141,91 @@ class ConnectivityTester:
return result
def _analyze_connectivity(self, result: Dict) -> Tuple[str, str]:
"""Analyze connectivity results and provide recommendations."""
hostname = result['hostname']
primary_ip = result['primary_ip']
fallback_ip = result['fallback_ip']
"""Analyze connectivity results and provide recommendations.
# Primary IP works perfectly
if result['primary_ping'] and result['primary_ssh']:
return 'success', f"{hostname} is fully accessible via primary IP {primary_ip}"
Split into smaller helpers to keep this function's complexity low
while preserving the original decision logic.
"""
for handler in (
self._handle_primary_success,
self._handle_primary_ping_only,
self._handle_fallback_path,
self._handle_no_fallback,
):
outcome = handler(result)
if outcome is not None:
return outcome
# Primary ping works but SSH fails
if result['primary_ping'] and not result['primary_ssh']:
error = result['primary_ssh_error']
if 'Permission denied' in error:
return 'ssh_key', f"{hostname}: SSH key issue on {primary_ip} - run: make copy-ssh-key HOST={hostname}"
elif 'Connection refused' in error:
return 'ssh_service', f"{hostname}: SSH service not running on {primary_ip}"
else:
return 'ssh_error', f"{hostname}: SSH error on {primary_ip} - {error}"
hostname = result["hostname"]
return "unknown", f"? {hostname}: Unknown connectivity state"
# Primary IP fails, test fallback
if not result['primary_ping'] and fallback_ip:
if result['fallback_ping'] and result['fallback_ssh']:
return 'use_fallback', f"{hostname}: Switch to fallback IP {fallback_ip} (primary {primary_ip} failed)"
elif result['fallback_ping'] and not result['fallback_ssh']:
return 'fallback_ssh', f"{hostname}: Fallback IP {fallback_ip} reachable but SSH failed"
else:
return 'both_failed', f"{hostname}: Both primary {primary_ip} and fallback {fallback_ip} failed"
def _handle_primary_success(self, result: Dict) -> Optional[Tuple[str, str]]:
"""Handle case where primary IP works perfectly."""
if result.get("primary_ping") and result.get("primary_ssh"):
hostname = result["hostname"]
primary_ip = result["primary_ip"]
return "success", f"{hostname} is fully accessible via primary IP {primary_ip}"
return None
# No fallback IP and primary failed
if not result['primary_ping'] and not fallback_ip:
return 'no_fallback', f"{hostname}: Primary IP {primary_ip} failed, no fallback available"
def _handle_primary_ping_only(self, result: Dict) -> Optional[Tuple[str, str]]:
"""Handle cases where primary ping works but SSH fails."""
if result.get("primary_ping") and not result.get("primary_ssh"):
hostname = result["hostname"]
primary_ip = result["primary_ip"]
error = result.get("primary_ssh_error", "")
return 'unknown', f"? {hostname}: Unknown connectivity state"
if "Permission denied" in error:
return (
"ssh_key",
f"{hostname}: SSH key issue on {primary_ip} - run: make copy-ssh-key HOST={hostname}",
)
if "Connection refused" in error:
return "ssh_service", f"{hostname}: SSH service not running on {primary_ip}"
return "ssh_error", f"{hostname}: SSH error on {primary_ip} - {error}"
return None
def _handle_fallback_path(self, result: Dict) -> Optional[Tuple[str, str]]:
"""Handle cases where primary fails and a fallback IP is defined."""
if result.get("primary_ping"):
return None
fallback_ip = result.get("fallback_ip")
if not fallback_ip:
return None
hostname = result["hostname"]
primary_ip = result["primary_ip"]
if result.get("fallback_ping") and result.get("fallback_ssh"):
return (
"use_fallback",
f"{hostname}: Switch to fallback IP {fallback_ip} (primary {primary_ip} failed)",
)
if result.get("fallback_ping") and not result.get("fallback_ssh"):
return (
"fallback_ssh",
f"{hostname}: Fallback IP {fallback_ip} reachable but SSH failed",
)
return (
"both_failed",
f"{hostname}: Both primary {primary_ip} and fallback {fallback_ip} failed",
)
def _handle_no_fallback(self, result: Dict) -> Optional[Tuple[str, str]]:
"""Handle cases where primary failed and no fallback IP is available."""
if result.get("primary_ping"):
return None
fallback_ip = result.get("fallback_ip")
if fallback_ip:
return None
hostname = result["hostname"]
primary_ip = result["primary_ip"]
return "no_fallback", f"{hostname}: Primary IP {primary_ip} failed, no fallback available"
def run_tests(self) -> List[Dict]:
"""Run connectivity tests for all hosts."""
@ -264,8 +316,8 @@ class ConnectivityTester:
# Auto-fallback suggestion
if fallback_needed:
print(f"\n🤖 Or run auto-fallback to fix automatically:")
print(f" make auto-fallback")
print("\n🤖 Or run auto-fallback to fix automatically:")
print(" make auto-fallback")
def export_json(self, results: List[Dict], output_file: str):
"""Export results to JSON file."""