kamp/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2024-11-11 02:10:11 +01:00
Code

Extends the checking of message*.properties (#1781)

Browse Source
This commit is contained in:
Ludy 2024-08-31 15:54:11 +02:00 committed by GitHub
parent 09e963b160
commit a14b78ff91
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 305 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
.github/scripts/check_language_properties.py vendored
View File

@ -1,5 +1,124 @@
"""
Author: Ludy87
Description: This script processes .properties files for localization checks. It compares translation files in a branch with
a reference file to ensure consistency. The script performs two main checks:
1. Verifies that the number of lines (including comments and empty lines) in the translation files matches the reference file.
2. Ensures that all keys in the translation files are present in the reference file and vice versa.
The script also provides functionality to update the translation files to match the reference file by adding missing keys and
adjusting the format.
Usage:
python script_name.py --reference-file <path_to_reference_file> --branch <branch_name> [--files <list_of_changed_files>]
"""
import copy
import glob
import os
import argparse
import re
def parse_properties_file(file_path):
"""Parses a .properties file and returns a list of objects (including comments, empty lines, and line numbers)."""
properties_list = []
with open(file_path, "r", encoding="utf-8") as file:
for line_number, line in enumerate(file, start=1):
stripped_line = line.strip()
# Empty lines
if not stripped_line:
properties_list.append(
{"line_number": line_number, "type": "empty", "content": ""}
)
continue
# Comments
if stripped_line.startswith("#"):
properties_list.append(
{
"line_number": line_number,
"type": "comment",
"content": stripped_line,
}
)
continue
# Key-value pairs
match = re.match(r"^([^=]+)=(.*)$", line)
if match:
key, value = match.groups()
properties_list.append(
{
"line_number": line_number,
"type": "entry",
"key": key.strip(),
"value": value.strip(),
}
)
return properties_list
def write_json_file(file_path, updated_properties):
updated_lines = {entry["line_number"]: entry for entry in updated_properties}
# Sort by line numbers and retain comments and empty lines
all_lines = sorted(set(updated_lines.keys()))
original_format = []
for line in all_lines:
if line in updated_lines:
entry = updated_lines[line]
else:
entry = None
ref_entry = updated_lines[line]
if ref_entry["type"] in ["comment", "empty"]:
original_format.append(ref_entry)
elif entry is None:
# Add missing entries from the reference file
original_format.append(ref_entry)
elif entry["type"] == "entry":
# Replace entries with those from the current JSON
original_format.append(entry)
# Write back in the original format
with open(file_path, "w", encoding="utf-8") as file:
for entry in original_format:
if entry["type"] == "comment":
file.write(f"{entry['content']}\n")
elif entry["type"] == "empty":
file.write(f"{entry['content']}\n")
elif entry["type"] == "entry":
file.write(f"{entry['key']}={entry['value']}\n")
def update_missing_keys(reference_file, file_list, branch=""):
reference_properties = parse_properties_file(reference_file)
for file_path in file_list:
basename_current_file = os.path.basename(branch + file_path)
if (
basename_current_file == os.path.basename(reference_file)
or not file_path.endswith(".properties")
or not basename_current_file.startswith("messages_")
):
continue
current_properties = parse_properties_file(branch + file_path)
updated_properties = []
for ref_entry in reference_properties:
ref_entry_copy = copy.deepcopy(ref_entry)
for current_entry in current_properties:
if current_entry["type"] == "entry":
if ref_entry_copy["type"] != "entry":
continue
if ref_entry_copy["key"] == current_entry["key"]:
ref_entry_copy["value"] = current_entry["value"]
updated_properties.append(ref_entry_copy)
write_json_file(branch + file_path, updated_properties)
def check_for_missing_keys(reference_file, file_list, branch):
update_missing_keys(reference_file, file_list, branch + "/")
def read_properties(file_path):
@ -7,87 +126,97 @@ def read_properties(file_path):
return file.read().splitlines()
def check_difference(reference_file, file_list, branch):
def check_for_differences(reference_file, file_list, branch):
reference_branch = reference_file.split("/")[0]
basename_reference_file = os.path.basename(reference_file)
report = []
report.append(
f"#### Checking with the file `{basename_reference_file}` from the `{reference_branch}` - Checking the `{branch}`"
f"### 📋 Checking with the file `{basename_reference_file}` from the `{reference_branch}` - Checking the `{branch}`"
)
reference_list = read_properties(reference_file)
is_diff = False
reference_lines = read_properties(reference_file)
has_differences = False
only_reference_file = True
for file_path in file_list:
basename_current_file = os.path.basename(branch + "/" + file_path)
if (
branch + "/" + file_path == reference_file
basename_current_file == basename_reference_file
or not file_path.endswith(".properties")
or not basename_current_file.startswith("messages_")
):
# report.append(f"File '{basename_current_file}' is ignored.")
continue
report.append(f"Checking the language file `{basename_current_file}`...")
current_list = read_properties(branch + "/" + file_path)
reference_list_len = len(reference_list)
current_list_len = len(current_list)
only_reference_file = False
report.append(f"#### 🗂️ **Checking File:** `{basename_current_file}`...")
current_lines = read_properties(branch + "/" + file_path)
reference_line_count = len(reference_lines)
current_line_count = len(current_lines)
if reference_list_len != current_list_len:
if reference_line_count != current_line_count:
report.append("")
report.append("- ❌ Test 1 failed! Difference in the file!")
is_diff = True
if reference_list_len > current_list_len:
report.append("- **Test 1 Status:** ❌ Failed")
has_differences = True
if reference_line_count > current_line_count:
report.append(
f" - Missing lines! Either comments, empty lines, or translation strings are missing! {reference_list_len}:{current_list_len}"
f" - **Issue:** Missing lines! Comments, empty lines, or translation strings are missing. Details: {reference_line_count} (reference) vs {current_line_count} (current)."
)
elif reference_list_len < current_list_len:
elif reference_line_count < current_line_count:
report.append(
f" - Too many lines! Check your translation files! {reference_list_len}:{current_list_len}"
f" - **Issue:** Too many lines! Check your translation files! Details: {reference_line_count} (reference) vs {current_line_count} (current)."
)
else:
report.append("- ✅ Test 1 passed")
if 1 == 1:
current_keys = []
reference_keys = []
for item in current_list:
if not item.startswith("#") and item != "" and "=" in item:
key, _ = item.split("=", 1)
current_keys.append(key)
for item in reference_list:
if not item.startswith("#") and item != "" and "=" in item:
key, _ = item.split("=", 1)
reference_keys.append(key)
report.append("- **Test 1 Status:** ✅ Passed")
current_set = set(current_keys)
reference_set = set(reference_keys)
set_test1 = current_set.difference(reference_set)
set_test2 = reference_set.difference(current_set)
set_test1_list = list(set_test1)
set_test2_list = list(set_test2)
# Check for missing or extra keys
current_keys = []
reference_keys = []
for line in current_lines:
if not line.startswith("#") and line != "" and "=" in line:
key, _ = line.split("=", 1)
current_keys.append(key)
for line in reference_lines:
if not line.startswith("#") and line != "" and "=" in line:
key, _ = line.split("=", 1)
reference_keys.append(key)
if len(set_test1_list) > 0 or len(set_test2_list) > 0:
is_diff = True
set_test1_list = "`, `".join(set_test1_list)
set_test2_list = "`, `".join(set_test2_list)
report.append("- ❌ Test 2 failed")
if len(set_test1_list) > 0:
report.append(
f" - There are keys in ***{basename_current_file}*** `{set_test1_list}` that are not present in ***{basename_reference_file}***!"
)
if len(set_test2_list) > 0:
report.append(
f" - There are keys in ***{basename_reference_file}*** `{set_test2_list}` that are not present in ***{basename_current_file}***!"
)
else:
report.append("- ✅ Test 2 passed")
current_keys_set = set(current_keys)
reference_keys_set = set(reference_keys)
missing_keys = current_keys_set.difference(reference_keys_set)
extra_keys = reference_keys_set.difference(current_keys_set)
missing_keys_list = list(missing_keys)
extra_keys_list = list(extra_keys)
if missing_keys_list or extra_keys_list:
has_differences = True
missing_keys_str = "`, `".join(missing_keys_list)
extra_keys_str = "`, `".join(extra_keys_list)
report.append("- **Test 2 Status:** ❌ Failed")
if missing_keys_list:
report.append(
f" - **Issue:** There are keys in ***{basename_current_file}*** `{missing_keys_str}` that are not present in ***{basename_reference_file}***!"
)
if extra_keys_list:
report.append(
f" - **Issue:** There are keys in ***{basename_reference_file}*** `{extra_keys_str}` that are not present in ***{basename_current_file}***!"
)
else:
report.append("- **Test 2 Status:** ✅ Passed")
if has_differences:
report.append("")
report.append(f"#### 🚧 ***{basename_current_file}*** will be corrected...")
report.append("")
report.append("---")
report.append("")
report.append("")
if is_diff:
report.append("## ❌ Check fail")
update_file_list = glob.glob(branch + "/src/**/messages_*.properties", recursive=True)
update_missing_keys(reference_file, update_file_list)
if has_differences:
report.append("## ❌ Overall Check Status: **_Failed_**")
else:
report.append("## ✅ Check success")
print("\n".join(report))
report.append("## ✅ Overall Check Status: **_Success_**")
if not only_reference_file:
print("\n".join(report))
if __name__ == "__main__":
@ -106,10 +235,16 @@ if __name__ == "__main__":
parser.add_argument(
"--files",
nargs="+",
required=True,
required=False,
help="List of changed files, separated by spaces.",
)
args = parser.parse_args()
file_list = args.files
check_difference(args.reference_file, file_list, args.branch)
if file_list is None:
file_list = glob.glob(
os.getcwd() + "/src/**/messages_*.properties", recursive=True
)
update_missing_keys(args.reference_file, file_list)
else:
check_for_differences(args.reference_file, file_list, args.branch)

121
.github/workflows/check_properties.yml vendored
View File

@ -1,15 +1,22 @@
name: Check Properties Files in PR
name: Check Properties Files
on:
pull_request_target:
types: [opened, synchronize, reopened]
paths:
- "src/main/resources/messages_*.properties"
push:
paths:
- "src/main/resources/messages_en_GB.properties"
permissions:
contents: write
pull-requests: write
jobs:
check-files:
if: github.event_name == 'pull_request_target'
runs-on: ubuntu-latest
steps:
- name: Checkout PR branch
uses: actions/checkout@v4
@ -54,11 +61,12 @@ jobs:
id: determine-file
run: |
echo "Determining reference file..."
if echo "${{ env.CHANGED_FILES }}"| grep -q 'src/main/resources/messages_en_GB.properties'; then
if echo "${{ env.CHANGED_FILES }}" | grep -q 'src/main/resources/messages_en_GB.properties'; then
echo "REFERENCE_FILE=pr-branch/src/main/resources/messages_en_GB.properties" >> $GITHUB_ENV
else
echo "REFERENCE_FILE=main-branch/src/main/resources/messages_en_GB.properties" >> $GITHUB_ENV
fi
echo "REFERENCE_FILE=${{ env.REFERENCE_FILE }}"
- name: Show REFERENCE_FILE
run: echo "Reference file is set to ${{ env.REFERENCE_FILE }}"
@ -71,26 +79,121 @@ jobs:
- name: Capture output
id: capture-output
run: |
if [ -f failure.txt ]; then
if [ -f failure.txt ] && [ -s failure.txt ]; then
echo "Test failed, capturing output..."
# Use the cat command to avoid issues with special characters in environment variables
ERROR_OUTPUT=$(cat failure.txt)
echo "ERROR_OUTPUT<<EOF" >> $GITHUB_ENV
echo "$ERROR_OUTPUT" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
echo $ERROR_OUTPUT
else
echo "No errors found."
echo "ERROR_OUTPUT=" >> $GITHUB_ENV
fi
- name: Post comment on PR
if: env.ERROR_OUTPUT != ''
uses: actions/github-script@v7
with:
script: |
const { GITHUB_REPOSITORY, ERROR_OUTPUT } = process.env;
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
const prNumber = context.issue.number; // Pull request number from context
await github.rest.issues.createComment({
const prNumber = context.issue.number;
// Find existing comment
const comments = await github.rest.issues.listComments({
owner: repoOwner,
repo: repoName,
issue_number: prNumber,
body: `## Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n`
issue_number: prNumber
});
const comment = comments.data.find(c => c.body.includes("## 🚀 Translation Verification Summary"));
// Only allow the action user to update comments
const expectedActor = "github-actions[bot]";
if (comment && comment.user.login === expectedActor) {
// Update existing comment
await github.rest.issues.updateComment({
owner: repoOwner,
repo: repoName,
comment_id: comment.id,
body: `## 🚀 Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n`
});
console.log("Updated existing comment.");
} else if (!comment) {
// Create new comment if no existing comment is found
await github.rest.issues.createComment({
owner: repoOwner,
repo: repoName,
issue_number: prNumber,
body: `## 🚀 Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n`
});
console.log("Created new comment.");
} else {
console.log("Comment update attempt denied. Actor does not match.");
}
- name: Set up git config
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
- name: Add translation keys
run: |
cd ${{ env.BRANCH_PATH }}
git add src/main/resources/messages_*.properties
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
git commit -m "Update translation files" || echo "No changes to commit"
- name: Push
if: env.CHANGES_DETECTED == 'true'
run: |
cd pr-branch
git push origin ${{ github.head_ref }} || echo "Push failed: possibly no changes to push"
update-translations-main:
if: github.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Run Python script to check files
id: run-check
run: |
python .github/scripts/check_language_properties.py --reference-file src/main/resources/messages_en_GB.properties --branch main
- name: Set up git config
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
- name: Add translation keys
run: |
git add src/main/resources/messages_*.properties
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
- name: Create Pull Request
id: cpr
if: env.CHANGES_DETECTED == 'true'
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: "Update translation files"
committer: GitHub Action <action@github.com>
author: GitHub Action <action@github.com>
signoff: true
branch: update_translation_files
title: "Update translation files"
body: |
Auto-generated by [create-pull-request][1]
[1]: https://github.com/peter-evans/create-pull-request
labels: Translation
draft: false
delete-branch: true