Skip to content

Commit 323cec3

Browse files
njhalenjhale
committed
test: add smoke test github workflow
1 parent 9ee00d4 commit 323cec3

File tree

2 files changed

+185
-1
lines changed

2 files changed

+185
-1
lines changed

‎.github/workflows/smoke.yaml

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
name: test
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
branches:
7+
- main
8+
- test/smoke
9+
push:
10+
branches:
11+
- main
12+
- test/smoke
13+
paths-ignore:
14+
- docs/**
15+
workflow_dispatch:
16+
17+
jobs:
18+
check-label:
19+
runs-on: ubuntu-22.04
20+
outputs:
21+
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
22+
steps:
23+
- name: Check if PR author is a member of the organization or has the run-smoke label
24+
id: check
25+
run: |
26+
case "${{ github.event_name }}" in
27+
push)
28+
# Run smoke tests for push to base repo
29+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
30+
exit 0
31+
;;
32+
workflow_dispatch)
33+
# Run smoke tests for manual runs against base branch
34+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
35+
exit 0
36+
;;
37+
pull_request_target)
38+
ORG="gptscript-ai"
39+
AUTHOR="${{ github.event.pull_request.user.login }}"
40+
41+
# Check for org membership
42+
MEMBERSHIP_RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
43+
"https://api.github.com/orgs/$ORG/members/$AUTHOR")
44+
45+
if [ "$MEMBERSHIP_RESPONSE_CODE" -eq 204 ]; then
46+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
47+
exit 0
48+
fi
49+
50+
# Check for "run-smoke" label
51+
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
52+
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
53+
if echo "$LABELS" | grep -q "run-smoke"; then
54+
# Run smoke tests for PR with the "run-smoke" label
55+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
56+
exit 0
57+
fi
58+
59+
;;
60+
esac
61+
62+
echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
63+
64+
smoke-gpt-4o-2024-05-13:
65+
needs: check-label
66+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
67+
runs-on: ubuntu-22.04
68+
steps:
69+
- name: Checkout base repository
70+
uses: actions/checkout@v4
71+
with:
72+
fetch-depth: 1
73+
- name: Checkout PR code if running for a PR
74+
if: ${{ github.event_name == 'pull_request_target' }}
75+
uses: actions/checkout@v4
76+
with:
77+
fetch-depth: 1
78+
repository: ${{ github.event.pull_request.head.repo.full_name }}
79+
ref: ${{ github.event.pull_request.head.ref }}
80+
- uses: actions/setup-go@v5
81+
with:
82+
cache: false
83+
go-version: "1.21"
84+
- env:
85+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
86+
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
87+
name: Run smoke test for gpt-4o-2024-05-13
88+
run: |
89+
echo "Running smoke test for model gpt-4o-2024-05-13"
90+
export PATH="$(pwd)/bin:${PATH}"
91+
make smoke
92+
93+
smoke-gpt-4-turbo-2024-04-09:
94+
needs: check-label
95+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
96+
runs-on: ubuntu-22.04
97+
steps:
98+
- name: Checkout base repository
99+
uses: actions/checkout@v4
100+
with:
101+
fetch-depth: 1
102+
- name: Checkout PR code if running for a PR
103+
if: ${{ github.event_name == 'pull_request_target' }}
104+
uses: actions/checkout@v4
105+
with:
106+
fetch-depth: 1
107+
repository: ${{ github.event.pull_request.head.repo.full_name }}
108+
ref: ${{ github.event.pull_request.head.ref }}
109+
- uses: actions/setup-go@v5
110+
with:
111+
cache: false
112+
go-version: "1.21"
113+
- env:
114+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
115+
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
116+
name: Run smoke test for gpt-4-turbo-2024-04-09
117+
run: |
118+
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
119+
export PATH="$(pwd)/bin:${PATH}"
120+
make smoke
121+
122+
smoke-claude-3-opus-20240229:
123+
needs: check-label
124+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
125+
runs-on: ubuntu-22.04
126+
steps:
127+
- name: Checkout base repository
128+
uses: actions/checkout@v4
129+
with:
130+
fetch-depth: 1
131+
- name: Checkout PR code if running for a PR
132+
if: ${{ github.event_name == 'pull_request_target' }}
133+
uses: actions/checkout@v4
134+
with:
135+
fetch-depth: 1
136+
repository: ${{ github.event.pull_request.head.repo.full_name }}
137+
ref: ${{ github.event.pull_request.head.ref }}
138+
- uses: actions/setup-go@v5
139+
with:
140+
cache: false
141+
go-version: "1.21"
142+
- env:
143+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
144+
GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
145+
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
146+
name: Run smoke test for claude-3-opus-20240229
147+
run: |
148+
echo "Running smoke test for model claude-3-opus-20240229"
149+
export PATH="$(pwd)/bin:${PATH}"
150+
make smoke
151+
152+
smoke-mistral-large-2402:
153+
needs: check-label
154+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
155+
runs-on: ubuntu-22.04
156+
steps:
157+
- name: Checkout base repository
158+
uses: actions/checkout@v4
159+
with:
160+
fetch-depth: 1
161+
- name: Checkout PR code if running for a PR
162+
if: ${{ github.event_name == 'pull_request_target' }}
163+
uses: actions/checkout@v4
164+
with:
165+
fetch-depth: 1
166+
repository: ${{ github.event.pull_request.head.repo.full_name }}
167+
ref: ${{ github.event.pull_request.head.ref }}
168+
- uses: actions/setup-go@v5
169+
with:
170+
cache: false
171+
go-version: "1.21"
172+
- env:
173+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
174+
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
175+
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
176+
name: Run smoke test for mistral-large-2402
177+
run: |
178+
echo "Running smoke test for model mistral-large-2402"
179+
export PATH="$(pwd)/bin:${PATH}"
180+
make smoke
181+

‎Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ tidy:
1414
test:
1515
go test -v ./...
1616

17+
smoke:
18+
go test -v -tags='smoke' ./pkg/tests/smoke/...
19+
1720
GOLANGCI_LINT_VERSION ?= v1.59.0
1821
lint:
1922
if ! command -v golangci-lint &> /dev/null; then \
@@ -52,4 +55,4 @@ validate-docs:
5255
echo "Encountered dirty repo!"; \
5356
git diff; \
5457
exit 1 \
55-
;fi
58+
;fi

0 commit comments

Comments
 (0)