-
Notifications
You must be signed in to change notification settings - Fork 101
196 lines (188 loc) · 6.76 KB
/
interop-test-suite.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
name: SOP interoperability test suite
on:
pull_request:
branches: [ main ]
jobs:
build-gosop-v1:
name: Build gosop from branch v1-api
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build gosop from branch
uses: ./.github/actions/build-gosop
with:
binary-location: ./gosop-${{ github.sha }}-v1
branch-gosop: gosop-gopenpgp-v2
# Upload as artifact
- name: Upload gosop artifact
uses: actions/upload-artifact@v3
with:
name: gosop-${{ github.sha }}-v1
path: ./gosop-${{ github.sha }}-v1
build-gosop-v2:
name: Build gosop from branch v2-api
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build gosop from branch
uses: ./.github/actions/build-gosop
with:
binary-location: ./gosop-${{ github.sha }}-v2
branch-gosop: gosop-gopenpgp-v3
gosop-build-path: build_gosop.sh
# Upload as artifact
- name: Upload gosop artifact
uses: actions/upload-artifact@v3
with:
name: gosop-${{ github.sha }}-v2
path: ./gosop-${{ github.sha }}-v2
build-gosop-main-v1:
name: Build gosop from main v1-api
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build gosop from branch
uses: ./.github/actions/build-gosop
with:
go-crypto-ref: main
branch-gosop: gosop-gopenpgp-v2
binary-location: ./gosop-main-v1
# Upload as artifact
- name: Upload gosop-main artifact
uses: actions/upload-artifact@v3
with:
name: gosop-main-v1
path: ./gosop-main-v1
build-gosop-main-v2:
name: Build gosop from main v2-api
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Build gosop from branch
uses: ./.github/actions/build-gosop
with:
go-crypto-ref: main
binary-location: ./gosop-main-v2
branch-gosop: gosop-gopenpgp-v3
gosop-build-path: build_gosop.sh
# Upload as artifact
- name: Upload gosop-main artifact
uses: actions/upload-artifact@v3
with:
name: gosop-main-v2
path: ./gosop-main-v2
test-suite:
name: Run interoperability test suite
runs-on: ubuntu-latest
container:
image: ghcr.io/protonmail/openpgp-interop-test-docker:v1.1.9
credentials:
username: ${{ github.actor }}
password: ${{ secrets.github_token }}
needs:
- build-gosop-v1
- build-gosop-v2
- build-gosop-main-v1
- build-gosop-main-v2
steps:
- name: Checkout
uses: actions/checkout@v3
# Fetch gosop from main v1
- name: Download gosop-main-v1
uses: actions/download-artifact@v3
with:
name: gosop-main-v1
# Test gosop-main-v1
- name: Make gosop-main executable
run: chmod +x gosop-main-v1
- name: Print gosop-main-v1 version
run: ./gosop-main-v1 version --extended
# Fetch gosop from main v2
- name: Download gosop-main-v2
uses: actions/download-artifact@v3
with:
name: gosop-main-v2
# Test gosop-main-v2
- name: Make gosop-main executable
run: chmod +x gosop-main-v2
- name: Print gosop-main-v2 version
run: ./gosop-main-v2 version --extended
# Fetch gosop from branch v1
- name: Download gosop-branch-v1
uses: actions/download-artifact@v3
with:
name: gosop-${{ github.sha }}-v1
- name: Rename gosop-branch-v1
run: mv gosop-${{ github.sha }}-v1 gosop-branch-v1
# Test gosop-branch v1
- name: Make gosop-branch-v1 executable
run: chmod +x gosop-branch-v1
- name: Print gosop-branch-v1 version
run: ./gosop-branch-v1 version --extended
# Fetch gosop from branch v2
- name: Download gosop-branch-v2
uses: actions/download-artifact@v3
with:
name: gosop-${{ github.sha }}-v2
- name: Rename gosop-branch-v2
run: mv gosop-${{ github.sha }}-v2 gosop-branch-v2
# Test gosop-branch v2
- name: Make gosop-branch-v2 executable
run: chmod +x gosop-branch-v2
- name: Print gosop-branch-v2 version
run: ./gosop-branch-v2 version --extended
# Run test suite
- name: Prepare test configuration
run: ./.github/test-suite/prepare_config.sh $CONFIG_TEMPLATE $CONFIG_OUTPUT $GITHUB_WORKSPACE/gosop-branch-v1 $GITHUB_WORKSPACE/gosop-branch-v2 $GITHUB_WORKSPACE/gosop-main-v1 $GITHUB_WORKSPACE/gosop-main-v2
env:
CONFIG_TEMPLATE: .github/test-suite/config.json.template
CONFIG_OUTPUT: .github/test-suite/config.json
- name: Display configuration
run: cat .github/test-suite/config.json
- name: Run interoperability test suite
run: cd $TEST_SUITE_DIR && $TEST_SUITE --config $GITHUB_WORKSPACE/$CONFIG --json-out $GITHUB_WORKSPACE/$RESULTS_JSON --html-out $GITHUB_WORKSPACE/$RESULTS_HTML
env:
CONFIG: .github/test-suite/config.json
RESULTS_JSON: .github/test-suite/test-suite-results.json
RESULTS_HTML: .github/test-suite/test-suite-results.html
# Upload results
- name: Upload test results json artifact
uses: actions/upload-artifact@v3
with:
name: test-suite-results.json
path: .github/test-suite/test-suite-results.json
- name: Upload test results html artifact
uses: actions/upload-artifact@v3
with:
name: test-suite-results.html
path: .github/test-suite/test-suite-results.html
compare-with-main:
name: Compare with main
runs-on: ubuntu-latest
needs: test-suite
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Download test results json artifact
id: download-test-results
uses: actions/download-artifact@v3
with:
name: test-suite-results.json
- name: Compare with baseline v1
uses: ProtonMail/openpgp-interop-test-analyzer@5d7f4b6868ebe3bfc909302828342c461f5f4940
with:
results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
output: baseline-comparison-v1.json
baseline: gosop-main-v1
target: gosop-branch-v1
- name: Compare with baseline v2
uses: ProtonMail/openpgp-interop-test-analyzer@5d7f4b6868ebe3bfc909302828342c461f5f4940
with:
results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
output: baseline-comparison-v2.json
baseline: gosop-main-v2
target: gosop-branch-v2