feat: Add new scripts and update project structure for database management and user authentication
This commit introduces several new scripts for managing database operations, including user creation, permission grants, and data migrations. It also adds new documentation files to guide users through the setup and configuration processes. Additionally, the project structure is updated to enhance organization and maintainability, ensuring a smoother development experience for contributors. These changes support the ongoing transition to a web-based architecture and improve overall project functionality.
This commit is contained in:
Tanya
@@ -0,0 +1,310 @@
|
||||
---
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, dev]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
# Check if CI should be skipped based on branch name or commit message
|
||||
skip-ci-check:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
should-skip: ${{ steps.check.outputs.skip }}
|
||||
steps:
|
||||
- name: Check out code (for commit message)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Check if CI should be skipped
|
||||
id: check
|
||||
run: |
|
||||
# Simple skip pattern: @skipci (case-insensitive)
|
||||
SKIP_PATTERN="@skipci"
|
||||
|
||||
# Get branch name (works for both push and PR)
|
||||
BRANCH_NAME="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
|
||||
|
||||
# Get commit message (works for both push and PR)
|
||||
COMMIT_MSG="${GITHUB_EVENT_HEAD_COMMIT_MESSAGE:-}"
|
||||
if [ -z "$COMMIT_MSG" ]; then
|
||||
COMMIT_MSG="${GITHUB_EVENT_PULL_REQUEST_HEAD_COMMIT_MESSAGE:-}"
|
||||
fi
|
||||
if [ -z "$COMMIT_MSG" ]; then
|
||||
COMMIT_MSG=$(git log -1 --pretty=%B 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
SKIP=0
|
||||
|
||||
# Check branch name (case-insensitive)
|
||||
if echo "$BRANCH_NAME" | grep -qiF "$SKIP_PATTERN"; then
|
||||
echo "Skipping CI: branch name contains '$SKIP_PATTERN'"
|
||||
SKIP=1
|
||||
fi
|
||||
|
||||
# Check commit message (case-insensitive)
|
||||
if [ $SKIP -eq 0 ] && [ -n "$COMMIT_MSG" ]; then
|
||||
if echo "$COMMIT_MSG" | grep -qiF "$SKIP_PATTERN"; then
|
||||
echo "Skipping CI: commit message contains '$SKIP_PATTERN'"
|
||||
SKIP=1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "skip=$SKIP" >> $GITHUB_OUTPUT
|
||||
echo "Branch: $BRANCH_NAME"
|
||||
echo "Commit: ${COMMIT_MSG:0:50}..."
|
||||
echo "Skip CI: $SKIP"
|
||||
|
||||
lint-and-type-check:
|
||||
needs: skip-ci-check
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
container:
|
||||
image: node:20-bullseye
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install admin-frontend dependencies
|
||||
run: |
|
||||
cd admin-frontend
|
||||
npm ci
|
||||
|
||||
- name: Run ESLint (admin-frontend)
|
||||
run: |
|
||||
cd admin-frontend
|
||||
npm run lint || true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Install viewer-frontend dependencies
|
||||
run: |
|
||||
cd viewer-frontend
|
||||
npm ci
|
||||
|
||||
- name: Type check (viewer-frontend)
|
||||
run: |
|
||||
cd viewer-frontend
|
||||
npm run type-check || true
|
||||
continue-on-error: true
|
||||
|
||||
python-lint:
|
||||
needs: skip-ci-check
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
container:
|
||||
image: python:3.12-slim
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
pip install --no-cache-dir flake8 black mypy pylint
|
||||
|
||||
- name: Check Python syntax
|
||||
run: |
|
||||
find backend -name "*.py" -exec python -m py_compile {} \; || true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run flake8
|
||||
run: |
|
||||
flake8 backend --max-line-length=100 --ignore=E501,W503 || true
|
||||
continue-on-error: true
|
||||
|
||||
test-backend:
|
||||
needs: skip-ci-check
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
container:
|
||||
image: python:3.12-slim
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: punimtag_test
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 6379:6379
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg2://postgres:postgres@postgres:5432/punimtag_test
|
||||
DATABASE_URL_AUTH: postgresql+psycopg2://postgres:postgres@postgres:5432/punimtag_auth_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
apt-get update && apt-get install -y postgresql-client
|
||||
pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
- name: Run backend tests
|
||||
run: |
|
||||
export PYTHONPATH=$(pwd)
|
||||
python -m pytest tests/ -v || true
|
||||
continue-on-error: true
|
||||
|
||||
build:
|
||||
needs: skip-ci-check
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
container:
|
||||
image: node:20-bullseye
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install admin-frontend dependencies
|
||||
run: |
|
||||
cd admin-frontend
|
||||
npm ci
|
||||
|
||||
- name: Build admin-frontend
|
||||
run: |
|
||||
cd admin-frontend
|
||||
npm run build
|
||||
env:
|
||||
VITE_API_URL: http://localhost:8000
|
||||
|
||||
- name: Install viewer-frontend dependencies
|
||||
run: |
|
||||
cd viewer-frontend
|
||||
npm ci
|
||||
|
||||
- name: Generate Prisma Client
|
||||
run: |
|
||||
cd viewer-frontend
|
||||
npx prisma generate
|
||||
|
||||
- name: Build viewer-frontend
|
||||
run: |
|
||||
cd viewer-frontend
|
||||
npm run build
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/punimtag
|
||||
DATABASE_URL_AUTH: postgresql://postgres:postgres@localhost:5432/punimtag_auth
|
||||
NEXTAUTH_SECRET: test-secret-key-for-ci
|
||||
NEXTAUTH_URL: http://localhost:3001
|
||||
|
||||
secret-scanning:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: zricethezav/gitleaks:latest
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apk add --no-cache nodejs npm curl
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Scan for secrets
|
||||
run: gitleaks detect --source . --no-banner --redact --exit-code 0
|
||||
continue-on-error: true
|
||||
|
||||
dependency-scan:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: aquasec/trivy:latest
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apk add --no-cache nodejs npm curl
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Dependency vulnerability scan (Trivy)
|
||||
run: |
|
||||
trivy fs \
|
||||
--scanners vuln \
|
||||
--severity HIGH,CRITICAL \
|
||||
--ignore-unfixed \
|
||||
--timeout 10m \
|
||||
--skip-dirs .git,node_modules,venv \
|
||||
--exit-code 0 \
|
||||
.
|
||||
|
||||
- name: Secret scan (Trivy)
|
||||
run: |
|
||||
trivy fs \
|
||||
--scanners secret \
|
||||
--timeout 10m \
|
||||
--skip-dirs .git,node_modules,venv \
|
||||
--exit-code 0 \
|
||||
.
|
||||
|
||||
sast-scan:
|
||||
needs: skip-ci-check
|
||||
if: needs.skip-ci-check.outputs.should-skip != '1'
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ubuntu:22.04
|
||||
steps:
|
||||
- name: Install Node.js for checkout action
|
||||
run: |
|
||||
apt-get update && apt-get install -y curl
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
||||
apt-get install -y nodejs
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Semgrep
|
||||
run: |
|
||||
apt-get update && apt-get install -y python3 python3-pip
|
||||
pip3 install semgrep
|
||||
|
||||
- name: Run Semgrep scan
|
||||
run: semgrep --config=auto --error
|
||||
continue-on-error: true
|
||||
|
||||
workflow-summary:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint-and-type-check, python-lint, test-backend, build, secret-scanning, dependency-scan, sast-scan]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Generate workflow summary
|
||||
run: |
|
||||
echo "## 🔍 CI Workflow Summary" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "### Job Results" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 📝 Lint & Type Check | ${{ needs.lint-and-type-check.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🐍 Python Lint | ${{ needs.python-lint.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🧪 Backend Tests | ${{ needs.test-backend.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🏗️ Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🔐 Secret Scanning | ${{ needs.secret-scanning.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 📦 Dependency Scan | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "| 🔍 SAST Scan | ${{ needs.sast-scan.result }} |" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "### 📊 Summary" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "" >> $GITHUB_STEP_SUMMARY || true
|
||||
echo "All security and validation checks have completed." >> $GITHUB_STEP_SUMMARY || true
|
||||
continue-on-error: true
|
||||
|
||||
Reference in New Issue
Block a user