Skip to content

Commit 9b9cfec

Browse files
ikolomiikolomi
committed
ORT WIP
Signed-off-by: ikolomi <[email protected]>
1 parent daa5205 commit 9b9cfec

File tree

4 files changed

+322
-1
lines changed

4 files changed

+322
-1
lines changed

‎.github/workflows/ort-new.yml‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
name: The OSS Review Toolkit (ORT) Ilia
2+
3+
on:
4+
schedule:
5+
- cron: '*/5 * * * *'
6+
7+
pull_request:
8+
9+
workflow_dispatch:
10+
inputs:
11+
branch-name:
12+
description: 'The branch to run against the ORT tool'
13+
required: true
14+
default: 'main'
15+
# commit-id:
16+
# description: 'Commit ID to run against the ORT tool'
17+
# required: true
18+
# default: 'HEAD'
19+
20+
jobs:
21+
run-ort:
22+
name: Create attribution files
23+
runs-on: ubuntu-latest
24+
# 1. For scheduled, allow only for the branch that triggered the WF and its eith main or release-*
25+
# 2. For workflow_dispatch, always allow
26+
# 3. For pull_request, run if target branch is the base (no 'branches' filters) and branch is not autogenerated ort-diff-for- branches
27+
if: >
28+
(github.event_name == 'schedule' && (startsWith(github.ref, 'refs/heads/') && (github.ref_name == 'main' || startsWith(github.ref_name, 'release-')))) ||
29+
github.event_name == 'workflow_dispatch' ||
30+
(github.event_name == 'pull_request' && !startsWith(github.ref, 'ort-diff-for-'))
31+
strategy:
32+
fail-fast: false
33+
env:
34+
PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON"
35+
NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE"
36+
RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST"
37+
JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA"
38+
INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch-name }}
39+
#INPUT_TARGET_COMMIT: ${{ github.event.inputs.commit-id }}
40+
EVENT_NAME: ${{ github.event_name }}
41+
42+
steps:
43+
# - name: Check mutually exclusive branch-name and commit-id
44+
# id: validate
45+
# run: |
46+
# if [[ -n "$INPUT_TARGET_BRANCH" && -n "$INPUT_TARGET_COMMIT" ]]; then
47+
# echo "Error: Both branch-name and commit-id are provided. Only one should be specified."
48+
# exit 1
49+
# elif [[ -z "$INPUT_TARGET_BRANCH" && -z "$INPUT_TARGET_COMMIT" ]]; then
50+
# echo "Error: Neither branch-name nor commit-id are provided. One must be specified."
51+
# exit 1
52+
# fi
53+
# env:
54+
# INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch-name }}
55+
# INPUT_TARGET_COMMIT: ${{ github.event.inputs.commit-id }}
56+
57+
# - name: Proceed with workflow
58+
# if: success()
59+
# run: echo "Inputs are valid. Proceeding with workflow execution..."
60+
61+
# - name: Scheduled runs only for main and "release-*" branches
62+
# if: >
63+
# github.event_name == 'schedule' &&
64+
# (github.ref_name == 'main' || startsWith(github.ref_name, 'release-'))
65+
# run: echo "This task runs on main and release-* branches during scheduled runs."
66+
67+
- name: Setup target branch and commit
68+
run: |
69+
if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then
70+
echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV
71+
elif [ "$EVENT_NAME" == 'pull_request' ]; then
72+
echo "TARGET_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV
73+
else
74+
echo "TARGET_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
75+
fi
76+
77+
- name: Checkout target branch
78+
uses: actions/checkout@v4
79+
with:
80+
ref: ${{ env.TARGET_BRANCH }}
81+
82+
# - name: Check if target commit exists in the branch
83+
# run: |
84+
# if git merge-base --is-ancestor $TARGET_COMMIT HEAD; then
85+
# echo "Commit $TARGET_COMMIT exists in $TARGET_BRANCH branch, continuing..."
86+
# else
87+
# echo "Commit $TARGET_COMMIT does not exist in the $TARGET_BRANCH branch."
88+
# exit 1
89+
# fi
90+
91+
- name: Setup target commit
92+
run: |
93+
echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV
94+
95+
- name: Set up JDK 11 for the ORT package
96+
uses: actions/setup-java@v4
97+
with:
98+
distribution: "temurin"
99+
java-version: 11
100+
101+
- name: Cache ORT and Gradle packages
102+
uses: actions/cache@v4
103+
id: cache-ort
104+
with:
105+
path: |
106+
./ort
107+
~/.gradle/caches
108+
~/.gradle/wrapper
109+
key: ${{ runner.os }}-ort
110+
111+
- name: Checkout ORT Repository
112+
if: steps.cache-ort.outputs.cache-hit != 'true'
113+
uses: actions/checkout@v4
114+
with:
115+
repository: "oss-review-toolkit/ort"
116+
path: "./ort"
117+
ref: "26.0.0"
118+
submodules: recursive
119+
120+
- name: Install Rust toolchain
121+
uses: dtolnay/[email protected]
122+
123+
- name: Build and install ORT
124+
if: steps.cache-ort.outputs.cache-hit != 'true'
125+
working-directory: ./ort/
126+
run: |
127+
export JAVA_OPTS="$JAVA_OPTS -Xmx8g"
128+
./gradlew installDist
129+
130+
- name: Create ORT config file
131+
run: |
132+
mkdir -p ~/.ort/config
133+
cat << EOF > ~/.ort/config/config.yml
134+
ort:
135+
analyzer:
136+
allowDynamicVersions: true
137+
enabledPackageManagers: [Cargo, NPM, PIP, GradleInspector]
138+
EOF
139+
cat ~/.ort/config/config.yml
140+
141+
### NodeJS ###
142+
- name: Set up Node.js 16.x
143+
uses: actions/setup-node@v4
144+
with:
145+
node-version: 16.x
146+
147+
- name: Create package.json file for the Node wrapper
148+
uses: ./.github/workflows/node-create-package-file
149+
with:
150+
release_version: ${{ env.TARGET_COMMIT }}
151+
os: "ubuntu-latest"
152+
153+
- name: Fix Node base NPM package.json file for ORT
154+
working-directory: ./node/npm/glide
155+
run: |
156+
# Remove the glide-rs dependency to avoid duplication
157+
sed -i '/ "glide-rs":/d' ../../package.json
158+
export pkg_name=valkey-glide-base
159+
export package_version="${{ env.TARGET_COMMIT }}"
160+
export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi`
161+
mv package.json package.json.tmpl
162+
envsubst < package.json.tmpl > "package.json"
163+
cat package.json
164+
165+
- name: Run ORT tools for Node
166+
uses: ./.github/workflows/run-ort-tools
167+
with:
168+
folder_path: "${{ github.workspace }}/node"
169+
170+
### Python ###
171+
- name: Set up Python 3.10
172+
uses: actions/setup-python@v5
173+
with:
174+
python-version: "3.10"
175+
176+
- name: Install python-inspector
177+
working-directory: ./python
178+
run: |
179+
python -m pip install --upgrade pip
180+
pip install git+https://github.com/nexB/python-inspector
181+
182+
- name: Run ORT tools for Python
183+
uses: ./.github/workflows/run-ort-tools
184+
with:
185+
folder_path: "${{ github.workspace }}/python"
186+
187+
### Rust glide-core ###
188+
- name: Run ORT tools for glide-core
189+
uses: ./.github/workflows/run-ort-tools
190+
with:
191+
folder_path: "${{ github.workspace }}/glide-core"
192+
193+
### Java ###
194+
- name: Set up JDK 11
195+
uses: actions/setup-java@v4
196+
with:
197+
distribution: "temurin"
198+
java-version: 11
199+
200+
- name: Run ORT tools for Java
201+
uses: ./.github/workflows/run-ort-tools
202+
with:
203+
folder_path: "${{ github.workspace }}/java"
204+
205+
206+
### Get licenses ###
207+
- name: Retrieve licenses list
208+
working-directory: ./utils
209+
run: |
210+
{
211+
echo 'LICENSES_LIST<<EOF'
212+
python3 get_licenses_from_ort.py
213+
echo EOF
214+
} >> "$GITHUB_ENV"
215+
216+
### Upload licenses ###
217+
- name: Get current date
218+
id: date
219+
run: |
220+
CURR_DATE=$(date +'%Y-%m-%d-%H')
221+
echo "date=${CURR_DATE}" >> $GITHUB_OUTPUT
222+
223+
- name: Upload the final package list
224+
continue-on-error: true
225+
uses: actions/upload-artifact@v4
226+
with:
227+
name: final-package-list-${{ steps.date.outputs.date }}
228+
path: |
229+
utils/final_package_list.txt
230+
retention-days: 30
231+
232+
- name: Upload the skipped package list
233+
continue-on-error: true
234+
uses: actions/upload-artifact@v4
235+
with:
236+
name: skipped-package-list-${{ steps.date.outputs.date }}
237+
path: |
238+
utils/skipped_package_list.txt
239+
retention-days: 30
240+
241+
- name: Upload the unknown/unapproved package list
242+
continue-on-error: true
243+
uses: actions/upload-artifact@v4
244+
with:
245+
name: unapproved-package-list-${{ steps.date.outputs.date }}
246+
path: |
247+
utils/unapproved_package_list.txt
248+
retention-days: 30
249+
250+
### TODO: Fail if there are unapproved packages ###
251+
252+
### Check for attributions diff ###
253+
- name: Check for diff
254+
run: |
255+
cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS
256+
cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS
257+
cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS
258+
cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS
259+
GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS`
260+
if [ -n "$GIT_DIFF" ]; then
261+
echo "FOUND_DIFF=true" >> $GITHUB_ENV
262+
else
263+
echo "FOUND_DIFF=false" >> $GITHUB_ENV
264+
fi
265+
266+
### Create PR, Note a potential race on the source branch ###
267+
- name: Create pull request
268+
if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }}
269+
run: |
270+
export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH"
271+
echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH"
272+
git config --global user.email "[email protected]"
273+
git config --global user.name "ort-bot"
274+
git checkout -b ${ORT_DIFF_BRANCH_NAME}
275+
git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS
276+
git commit -m "Updated attribution files" -s
277+
git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f
278+
279+
# Check if PR already exists
280+
existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number')
281+
282+
if [ -z "$existing_pr" ]; then
283+
# Create a new PR if none exists
284+
title="Updated attribution files for commit ${TARGET_COMMIT}"
285+
gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
286+
echo "Pull request created successfully."
287+
else
288+
# Update the existing PR
289+
echo "Pull request #$existing_pr already exists. Updating branch."
290+
gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
291+
echo "Pull request updated successfully."
292+
fi
293+
env:
294+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
295+
INPUT_VERSION: ${{ github.event.inputs.version }}
296+
297+
### Warn of outdated attributions for PR ###
298+
- name: Warn of outdated attributions due to the PR
299+
if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }}
300+
run: |
301+
YELLOW='\033[1;33m'
302+
NC='\033[0m'
303+
echo -e "${YELLOW}WARNING! Note the attribution files differ with this PR, make sure an updating PR is issued using scheduled or manual run of this workflow!${NC}"

‎glide-core/redis-rs/Cargo.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# Dummy package so ORT tool will not fail on virtual workspace
2+
[package]
3+
name = "dummy-for-ort"
4+
version = "0.1.0"
5+
edition = "2021"
6+
17
[workspace]
2-
members = ["redis", "redis-test"]
8+
members = ["redis", "redis-test", "afl/parser"]
39
resolver = "2"

‎glide-core/redis-rs/src/main.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
fn main() {
2+
println!("Dummy source to bypass ORT OSS Tool virtual workspace restrictions.");
3+
}

‎utils/get_licenses_from_ort.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import json
44
import os
55
from typing import List, Optional, Set
6+
import sys
67

78
"""
89
This script should be used after all specific langauge folders were scanned by the analyzer of the OSS review tool (ORT).
@@ -124,6 +125,10 @@ def __str__(self):
124125
with open(skipped_list_file_path, mode="wt", encoding="utf-8") as f:
125126
f.writelines(f"{package}\n" for package in skipped_packages)
126127

128+
unapproved_list_file_path = f"{SCRIPT_PATH}/unapproved_package_list.txt"
129+
with open(unapproved_list_file_path, mode="wt", encoding="utf-8") as f:
130+
f.writelines(f"{package}\n" for package in unknown_licenses)
131+
127132
print("\n\n#### Found Licenses #####\n")
128133
all_licenses_set = set(sorted(all_licenses_set))
129134
for license in all_licenses_set:
@@ -132,3 +137,7 @@ def __str__(self):
132137
print("\n\n#### unknown / Not Pre-Approved Licenses #####\n")
133138
for package in unknown_licenses:
134139
print(str(package))
140+
141+
# if unknown_licenses:
142+
# print("Unknown or unapproved licenses detected!")
143+
# sys.exit(1)

0 commit comments

Comments
 (0)