diff --git a/.github/workflows/_collect.yml b/.github/workflows/_collect.yml new file mode 100644 index 0000000..92b7512 --- /dev/null +++ b/.github/workflows/_collect.yml @@ -0,0 +1,92 @@ +name: Collect (reusable) + +on: + workflow_call: + inputs: + environment: + description: "GitHub Environment (dev / prod)" + type: string + required: true + ref: + description: "수집에 쓸 브랜치" + type: string + required: true + +permissions: + contents: read + +jobs: + # 테스트가 깨진 코드로는 DB에 쓰지 않는다. + ci: + uses: ./.github/workflows/ci.yml + with: + ref: ${{ inputs.ref }} + + collect: + name: Collect into ${{ inputs.environment }} + needs: ci + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + timeout-minutes: 150 + concurrency: + group: collect-${{ inputs.environment }} + cancel-in-progress: false + permissions: + contents: read + id-token: write # GCP OIDC + env: + GCE_INSTANCE: ${{ vars.GCE_INSTANCE }} + GCE_ZONE: ${{ vars.GCE_ZONE }} + steps: + - name: Checkout sources + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Set up Java 21 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v6 + + - name: Install Chromium for bank URL scraping + run: | + bash ./gradlew playwrightInstall + npx -y playwright@1.61.0 install-deps chromium + + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + project_id: ${{ vars.GCP_PROJECT }} + workload_identity_provider: ${{ vars.GCP_WIF_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v3 + + - name: Open DB tunnel + # 서버 Postgres는 127.0.0.1:5678에만 열려 있다. IAP SSH 포트 포워딩으로 러너 15432에 연결한다. + run: | + gcloud compute ssh "$GCE_INSTANCE" --zone "$GCE_ZONE" --tunnel-through-iap --quiet \ + --ssh-key-expire-after=1h \ + -- -f -N -o ExitOnForwardFailure=yes -L 15432:127.0.0.1:5678 + for _ in $(seq 1 30); do + (echo > /dev/tcp/127.0.0.1/15432) 2>/dev/null && exit 0 + sleep 1 + done + echo "DB tunnel did not open" >&2 + exit 1 + + - name: Run collector + env: + SPRING_DATASOURCE_URL: jdbc:postgresql://127.0.0.1:15432/database + SPRING_DATASOURCE_USERNAME: user + SPRING_DATASOURCE_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + FSS_API_KEY: ${{ secrets.FSS_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + LLM_ENABLED: "true" + # 배치가 FAILED면 0이 아닌 코드로 끝나 이 단계가 실패한다(ApiCollectorApplication.main). + run: bash ./gradlew bootRun diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml new file mode 100644 index 0000000..37597dc --- /dev/null +++ b/.github/workflows/collect.yml @@ -0,0 +1,40 @@ +name: Collect + +# 금융상품 수집 배치를 Actions 러너에서 돌려 서버 DB에 쓴다(서버 e2-micro에는 Chromium을 띄울 여유가 없다). +# main → dev DB, release → 운영 DB. 설정은 CLAUDE.md의 "수집 워크플로" 참고. +on: + schedule: + # LLM API가 유료라 약 2주 간격. 매월 1·15일 19:00 UTC = 2·16일 04:00 KST + - cron: "0 19 1,15 * *" + workflow_dispatch: + inputs: + environment: + description: "수집 대상 환경" + type: choice + options: [dev, prod] + default: dev + +permissions: + contents: read + +jobs: + dev: + if: github.event_name == 'schedule' || inputs.environment == 'dev' + uses: ./.github/workflows/_collect.yml + permissions: + contents: read + id-token: write + with: + environment: dev + ref: main + + prod: + # 운영 서버와 release 브랜치가 준비되면 레포 변수 PROD_ENABLED=true로 켠다. + if: vars.PROD_ENABLED == 'true' && (github.event_name == 'schedule' || inputs.environment == 'prod') + uses: ./.github/workflows/_collect.yml + permissions: + contents: read + id-token: write + with: + environment: prod + ref: release diff --git a/CLAUDE.md b/CLAUDE.md index 77cb2d5..b44423e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,3 +34,14 @@ api-collector main에 박히는 포인터는 **Fin-BE main에 머지된 커밋** 1. Fin-BE 스키마 변경을 main에 머지 2. `git submodule update --remote fin-be` → 포인터 bump 후 커밋 + +## 수집 워크플로 (`.github/workflows/collect.yml`) + +수집 배치는 서버가 아니라 GitHub Actions 러너에서 돈다(서버 e2-micro에는 Chromium을 띄울 여유가 없음). + +- 주기: LLM API 비용 때문에 약 2주 간격(매월 2·16일 04:00 KST). 그 사이에는 Actions에서 수동 실행(`workflow_dispatch`). +- `main` → dev DB, `release` → 운영 DB. 운영은 레포 변수 `PROD_ENABLED=true`일 때만 돈다. +- 수집 전에 같은 브랜치로 `ci.yml`(테스트)을 돌리고, 통과해야 수집한다. +- 서버 접속: GCP OIDC 인증 → IAP SSH 포트 포워딩으로 서버 Postgres(`127.0.0.1:5678`)에 붙는다. 서버 구성과 GCP 설정은 Fin-BE `docs/implementation/deployment.md`. +- GitHub Environment(`dev`, `prod`): vars `GCP_PROJECT`, `GCP_WIF_PROVIDER`, `GCP_SERVICE_ACCOUNT`, `GCE_INSTANCE`, `GCE_ZONE` / secrets `POSTGRES_PASSWORD`, `FSS_API_KEY`, `GEMINI_API_KEY`. +- **순서 제약**: 엔티티가 새 컬럼을 쓰면 Fin-BE가 그 마이그레이션을 해당 환경에 먼저 배포해야 한다. 아니면 `ddl-auto: validate`에서 수집이 실패한다.