Skip to content

Commit 641c682

Browse files
njhalenjhale
committed
test: add smoke test github workflow
1 parent 9ee00d4 commit 641c682

File tree

2 files changed

+183
-1
lines changed

2 files changed

+183
-1
lines changed

‎.github/workflows/smoke.yaml

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
name: test
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, synchronize, reopened]
6+
branches:
7+
- main
8+
push:
9+
branches:
10+
- main
11+
paths-ignore:
12+
- docs/**
13+
workflow_dispatch:
14+
15+
jobs:
16+
check-label:
17+
runs-on: ubuntu-22.04
18+
outputs:
19+
run_smoke_tests: ${{ steps.check.outputs.run_smoke_tests }}
20+
steps:
21+
- name: Check if PR author is a member of the organization or has the run-smoke label
22+
id: check
23+
run: |
24+
case "${{ github.event_name }}" in
25+
push)
26+
# Run smoke tests for push to base repo
27+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
28+
exit 0
29+
;;
30+
workflow_dispatch)
31+
# Run smoke tests for manual runs against base branch
32+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
33+
exit 0
34+
;;
35+
pull_request_target)
36+
ORG="gptscript-ai"
37+
AUTHOR="${{ github.event.pull_request.user.login }}"
38+
39+
# Check for org membership
40+
MEMBERSHIP_RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
41+
"https://api.github.com/orgs/$ORG/members/$AUTHOR")
42+
43+
if [ "$MEMBERSHIP_RESPONSE_CODE" -eq 204 ]; then
44+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
45+
exit 0
46+
fi
47+
48+
# Check for "run-smoke" label
49+
LABELS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
50+
"https://api.github.com/repos/${{ github.repository_owner }}/${{ github.event.repository.name }}/issues/${{ github.event.pull_request.number }}/labels" | jq -r '.[].name')
51+
if echo "$LABELS" | grep -q "run-smoke"; then
52+
# Run smoke tests for PR with the "run-smoke" label
53+
echo "run_smoke_tests=true" >> $GITHUB_OUTPUT
54+
exit 0
55+
fi
56+
57+
;;
58+
esac
59+
60+
echo "run_smoke_tests=false" >> $GITHUB_OUTPUT
61+
62+
smoke-gpt-4o-2024-05-13:
63+
needs: check-label
64+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
65+
runs-on: ubuntu-22.04
66+
steps:
67+
- name: Checkout base repository
68+
uses: actions/checkout@v4
69+
with:
70+
fetch-depth: 1
71+
- name: Checkout PR code if running for a PR
72+
if: ${{ github.event_name == 'pull_request_target' }}
73+
uses: actions/checkout@v4
74+
with:
75+
fetch-depth: 1
76+
repository: ${{ github.event.pull_request.head.repo.full_name }}
77+
ref: ${{ github.event.pull_request.head.ref }}
78+
- uses: actions/setup-go@v5
79+
with:
80+
cache: false
81+
go-version: "1.21"
82+
- env:
83+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
84+
GPTSCRIPT_DEFAULT_MODEL: gpt-4o-2024-05-13
85+
name: Run smoke test for gpt-4o-2024-05-13
86+
run: |
87+
echo "Running smoke test for model gpt-4o-2024-05-13"
88+
export PATH="$(pwd)/bin:${PATH}"
89+
make smoke
90+
91+
smoke-gpt-4-turbo-2024-04-09:
92+
needs: check-label
93+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
94+
runs-on: ubuntu-22.04
95+
steps:
96+
- name: Checkout base repository
97+
uses: actions/checkout@v4
98+
with:
99+
fetch-depth: 1
100+
- name: Checkout PR code if running for a PR
101+
if: ${{ github.event_name == 'pull_request_target' }}
102+
uses: actions/checkout@v4
103+
with:
104+
fetch-depth: 1
105+
repository: ${{ github.event.pull_request.head.repo.full_name }}
106+
ref: ${{ github.event.pull_request.head.ref }}
107+
- uses: actions/setup-go@v5
108+
with:
109+
cache: false
110+
go-version: "1.21"
111+
- env:
112+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
113+
GPTSCRIPT_DEFAULT_MODEL: gpt-4-turbo-2024-04-09
114+
name: Run smoke test for gpt-4-turbo-2024-04-09
115+
run: |
116+
echo "Running smoke test for model gpt-4-turbo-2024-04-09"
117+
export PATH="$(pwd)/bin:${PATH}"
118+
make smoke
119+
120+
smoke-claude-3-opus-20240229:
121+
needs: check-label
122+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
123+
runs-on: ubuntu-22.04
124+
steps:
125+
- name: Checkout base repository
126+
uses: actions/checkout@v4
127+
with:
128+
fetch-depth: 1
129+
- name: Checkout PR code if running for a PR
130+
if: ${{ github.event_name == 'pull_request_target' }}
131+
uses: actions/checkout@v4
132+
with:
133+
fetch-depth: 1
134+
repository: ${{ github.event.pull_request.head.repo.full_name }}
135+
ref: ${{ github.event.pull_request.head.ref }}
136+
- uses: actions/setup-go@v5
137+
with:
138+
cache: false
139+
go-version: "1.21"
140+
- env:
141+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
142+
GPTSCRIPT_DEFAULT_MODEL: claude-3-opus-20240229 from github.com/gptscript-ai/claude3-anthropic-provider@tool-beta
143+
ANTHROPIC_API_KEY: ${{ secrets.SMOKE_ANTHROPIC_API_KEY }}
144+
name: Run smoke test for claude-3-opus-20240229
145+
run: |
146+
echo "Running smoke test for model claude-3-opus-20240229"
147+
export PATH="$(pwd)/bin:${PATH}"
148+
make smoke
149+
150+
smoke-mistral-large-2402:
151+
needs: check-label
152+
if: ${{ needs.check-label.outputs.run_smoke_tests == 'true' }}
153+
runs-on: ubuntu-22.04
154+
steps:
155+
- name: Checkout base repository
156+
uses: actions/checkout@v4
157+
with:
158+
fetch-depth: 1
159+
- name: Checkout PR code if running for a PR
160+
if: ${{ github.event_name == 'pull_request_target' }}
161+
uses: actions/checkout@v4
162+
with:
163+
fetch-depth: 1
164+
repository: ${{ github.event.pull_request.head.repo.full_name }}
165+
ref: ${{ github.event.pull_request.head.ref }}
166+
- uses: actions/setup-go@v5
167+
with:
168+
cache: false
169+
go-version: "1.21"
170+
- env:
171+
OPENAI_API_KEY: ${{ secrets.SMOKE_OPENAI_API_KEY }}
172+
GPTSCRIPT_DEFAULT_MODEL: mistral-large-2402 from https://api.mistral.ai/v1
173+
GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY: ${{ secrets.SMOKE_GPTSCRIPT_PROVIDER_API_MISTRAL_AI_API_KEY }}
174+
name: Run smoke test for mistral-large-2402
175+
run: |
176+
echo "Running smoke test for model mistral-large-2402"
177+
export PATH="$(pwd)/bin:${PATH}"
178+
make smoke
179+

‎Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ tidy:
1414
test:
1515
go test -v ./...
1616

17+
smoke:
18+
go test -v -tags='smoke' ./pkg/tests/smoke/...
19+
1720
GOLANGCI_LINT_VERSION ?= v1.59.0
1821
lint:
1922
if ! command -v golangci-lint &> /dev/null; then \
@@ -52,4 +55,4 @@ validate-docs:
5255
echo "Encountered dirty repo!"; \
5356
git diff; \
5457
exit 1 \
55-
;fi
58+
;fi

0 commit comments

Comments
 (0)