From 5decddb75b33b4fb0b4b82f8071f02205e38210f Mon Sep 17 00:00:00 2001 From: yuyeol3 Date: Mon, 26 Oct 2026 16:32:51 +0900 Subject: [PATCH 1/2] =?UTF-8?q?chore=20:=202=EC=A3=BC=20=EA=B0=84=EA=B2=A9?= =?UTF-8?q?=20=EC=88=98=EC=A7=91=20=EC=9B=8C=ED=81=AC=ED=94=8C=EB=A1=9C=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80=20(main=E2=86=92dev,=20release=E2=86=92?= =?UTF-8?q?=EC=9A=B4=EC=98=81=20DB)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actions 러너에서 CI 통과 후 GCP OIDC + IAP 터널로 서버 DB에 붙어 수집 배치를 돌린다 --- .github/workflows/_collect.yml | 91 ++++++++++++++++++++++++++++++++++ .github/workflows/collect.yml | 40 +++++++++++++++ CLAUDE.md | 11 ++++ 3 files changed, 142 insertions(+) create mode 100644 .github/workflows/_collect.yml create mode 100644 .github/workflows/collect.yml diff --git a/.github/workflows/_collect.yml b/.github/workflows/_collect.yml new file mode 100644 index 0000000..00eb04f --- /dev/null +++ b/.github/workflows/_collect.yml @@ -0,0 +1,91 @@ +name: Collect (reusable) + +on: + workflow_call: + inputs: + environment: + description: "GitHub Environment (dev / prod)" + type: string + required: true + ref: + description: "수집에 쓸 브랜치" + type: string + required: true + +permissions: + contents: read + +jobs: + # 테스트가 깨진 코드로는 DB에 쓰지 않는다. + ci: + uses: ./.github/workflows/ci.yml + with: + ref: ${{ inputs.ref }} + + collect: + name: Collect into ${{ inputs.environment }} + needs: ci + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + timeout-minutes: 150 + concurrency: + group: collect-${{ inputs.environment }} + cancel-in-progress: false + permissions: + contents: read + id-token: write # GCP OIDC + env: + GCE_INSTANCE: ${{ vars.GCE_INSTANCE }} + GCE_ZONE: ${{ vars.GCE_ZONE }} + steps: + - name: Checkout sources + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Set up Java 21 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + + - name: Set up Gradle + uses: gradle/actions/setup-gradle@v6 + + - name: Install Chromium for bank URL scraping + run: | + bash ./gradlew playwrightInstall + npx -y playwright@1.61.0 install-deps chromium + + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + project_id: ${{ vars.GCP_PROJECT }} + workload_identity_provider: ${{ vars.GCP_WIF_PROVIDER }} + service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v3 + + - name: Open DB tunnel + # 서버 Postgres는 127.0.0.1:5678에만 열려 있다. IAP SSH 포트 포워딩으로 러너 15432에 연결한다. + run: | + gcloud compute ssh "$GCE_INSTANCE" --zone "$GCE_ZONE" --tunnel-through-iap --quiet \ + -- -f -N -o ExitOnForwardFailure=yes -L 15432:127.0.0.1:5678 + for _ in $(seq 1 30); do + (echo > /dev/tcp/127.0.0.1/15432) 2>/dev/null && exit 0 + sleep 1 + done + echo "DB tunnel did not open" >&2 + exit 1 + + - name: Run collector + env: + SPRING_DATASOURCE_URL: jdbc:postgresql://127.0.0.1:15432/database + SPRING_DATASOURCE_USERNAME: user + SPRING_DATASOURCE_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + FSS_API_KEY: ${{ secrets.FSS_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + LLM_ENABLED: "true" + # 배치가 FAILED면 0이 아닌 코드로 끝나 이 단계가 실패한다(ApiCollectorApplication.main). + run: bash ./gradlew bootRun diff --git a/.github/workflows/collect.yml b/.github/workflows/collect.yml new file mode 100644 index 0000000..37597dc --- /dev/null +++ b/.github/workflows/collect.yml @@ -0,0 +1,40 @@ +name: Collect + +# 금융상품 수집 배치를 Actions 러너에서 돌려 서버 DB에 쓴다(서버 e2-micro에는 Chromium을 띄울 여유가 없다). +# main → dev DB, release → 운영 DB. 설정은 CLAUDE.md의 "수집 워크플로" 참고. +on: + schedule: + # LLM API가 유료라 약 2주 간격. 매월 1·15일 19:00 UTC = 2·16일 04:00 KST + - cron: "0 19 1,15 * *" + workflow_dispatch: + inputs: + environment: + description: "수집 대상 환경" + type: choice + options: [dev, prod] + default: dev + +permissions: + contents: read + +jobs: + dev: + if: github.event_name == 'schedule' || inputs.environment == 'dev' + uses: ./.github/workflows/_collect.yml + permissions: + contents: read + id-token: write + with: + environment: dev + ref: main + + prod: + # 운영 서버와 release 브랜치가 준비되면 레포 변수 PROD_ENABLED=true로 켠다. + if: vars.PROD_ENABLED == 'true' && (github.event_name == 'schedule' || inputs.environment == 'prod') + uses: ./.github/workflows/_collect.yml + permissions: + contents: read + id-token: write + with: + environment: prod + ref: release diff --git a/CLAUDE.md b/CLAUDE.md index 77cb2d5..b44423e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,3 +34,14 @@ api-collector main에 박히는 포인터는 **Fin-BE main에 머지된 커밋** 1. Fin-BE 스키마 변경을 main에 머지 2. `git submodule update --remote fin-be` → 포인터 bump 후 커밋 + +## 수집 워크플로 (`.github/workflows/collect.yml`) + +수집 배치는 서버가 아니라 GitHub Actions 러너에서 돈다(서버 e2-micro에는 Chromium을 띄울 여유가 없음). + +- 주기: LLM API 비용 때문에 약 2주 간격(매월 2·16일 04:00 KST). 그 사이에는 Actions에서 수동 실행(`workflow_dispatch`). +- `main` → dev DB, `release` → 운영 DB. 운영은 레포 변수 `PROD_ENABLED=true`일 때만 돈다. +- 수집 전에 같은 브랜치로 `ci.yml`(테스트)을 돌리고, 통과해야 수집한다. +- 서버 접속: GCP OIDC 인증 → IAP SSH 포트 포워딩으로 서버 Postgres(`127.0.0.1:5678`)에 붙는다. 서버 구성과 GCP 설정은 Fin-BE `docs/implementation/deployment.md`. +- GitHub Environment(`dev`, `prod`): vars `GCP_PROJECT`, `GCP_WIF_PROVIDER`, `GCP_SERVICE_ACCOUNT`, `GCE_INSTANCE`, `GCE_ZONE` / secrets `POSTGRES_PASSWORD`, `FSS_API_KEY`, `GEMINI_API_KEY`. +- **순서 제약**: 엔티티가 새 컬럼을 쓰면 Fin-BE가 그 마이그레이션을 해당 환경에 먼저 배포해야 한다. 아니면 `ddl-auto: validate`에서 수집이 실패한다. From 8fb38227ce3d4f2a8b0326f9f2310bd40d4f6dbb Mon Sep 17 00:00:00 2001 From: yuyeol3 Date: Mon, 26 Oct 2026 16:47:22 +0900 Subject: [PATCH 2/2] =?UTF-8?q?chore=20:=20=EC=88=98=EC=A7=91=20=ED=84=B0?= =?UTF-8?q?=EB=84=90=EC=9A=A9=20SSH=20=ED=82=A4=201=EC=8B=9C=EA=B0=84=20?= =?UTF-8?q?=EB=A7=8C=EB=A3=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/_collect.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_collect.yml b/.github/workflows/_collect.yml index 00eb04f..92b7512 100644 --- a/.github/workflows/_collect.yml +++ b/.github/workflows/_collect.yml @@ -71,6 +71,7 @@ jobs: # 서버 Postgres는 127.0.0.1:5678에만 열려 있다. IAP SSH 포트 포워딩으로 러너 15432에 연결한다. run: | gcloud compute ssh "$GCE_INSTANCE" --zone "$GCE_ZONE" --tunnel-through-iap --quiet \ + --ssh-key-expire-after=1h \ -- -f -N -o ExitOnForwardFailure=yes -L 15432:127.0.0.1:5678 for _ in $(seq 1 30); do (echo > /dev/tcp/127.0.0.1/15432) 2>/dev/null && exit 0