From a14b78ff917be6d111c1be91203e639e2bba5f0e Mon Sep 17 00:00:00 2001 From: Ludy Date: Sat, 31 Aug 2024 15:54:11 +0200 Subject: [PATCH] Extends the checking of message*.properties (#1781) --- .github/scripts/check_language_properties.py | 251 ++++++++++++++----- .github/workflows/check_properties.yml | 121 ++++++++- 2 files changed, 305 insertions(+), 67 deletions(-) diff --git a/.github/scripts/check_language_properties.py b/.github/scripts/check_language_properties.py index bf6d6569..235b6f57 100644 --- a/.github/scripts/check_language_properties.py +++ b/.github/scripts/check_language_properties.py @@ -1,5 +1,124 @@ +""" +Author: Ludy87 +Description: This script processes .properties files for localization checks. It compares translation files in a branch with +a reference file to ensure consistency. The script performs two main checks: +1. Verifies that the number of lines (including comments and empty lines) in the translation files matches the reference file. +2. Ensures that all keys in the translation files are present in the reference file and vice versa. + +The script also provides functionality to update the translation files to match the reference file by adding missing keys and +adjusting the format. + +Usage: + python script_name.py --reference-file --branch [--files ] +""" +import copy +import glob import os import argparse +import re + + +def parse_properties_file(file_path): + """Parses a .properties file and returns a list of objects (including comments, empty lines, and line numbers).""" + properties_list = [] + with open(file_path, "r", encoding="utf-8") as file: + for line_number, line in enumerate(file, start=1): + stripped_line = line.strip() + + # Empty lines + if not stripped_line: + properties_list.append( + {"line_number": line_number, "type": "empty", "content": ""} + ) + continue + + # Comments + if stripped_line.startswith("#"): + properties_list.append( + { + "line_number": line_number, + "type": "comment", + "content": stripped_line, + } + ) + continue + + # Key-value pairs + match = re.match(r"^([^=]+)=(.*)$", line) + if match: + key, value = match.groups() + properties_list.append( + { + "line_number": line_number, + "type": "entry", + "key": key.strip(), + "value": value.strip(), + } + ) + + return properties_list + + +def write_json_file(file_path, updated_properties): + updated_lines = {entry["line_number"]: entry for entry in updated_properties} + + # Sort by line numbers and retain comments and empty lines + all_lines = sorted(set(updated_lines.keys())) + + original_format = [] + for line in all_lines: + if line in updated_lines: + entry = updated_lines[line] + else: + entry = None + ref_entry = updated_lines[line] + if ref_entry["type"] in ["comment", "empty"]: + original_format.append(ref_entry) + elif entry is None: + # Add missing entries from the reference file + original_format.append(ref_entry) + elif entry["type"] == "entry": + # Replace entries with those from the current JSON + original_format.append(entry) + + # Write back in the original format + with open(file_path, "w", encoding="utf-8") as file: + for entry in original_format: + if entry["type"] == "comment": + file.write(f"{entry['content']}\n") + elif entry["type"] == "empty": + file.write(f"{entry['content']}\n") + elif entry["type"] == "entry": + file.write(f"{entry['key']}={entry['value']}\n") + + +def update_missing_keys(reference_file, file_list, branch=""): + reference_properties = parse_properties_file(reference_file) + for file_path in file_list: + basename_current_file = os.path.basename(branch + file_path) + if ( + basename_current_file == os.path.basename(reference_file) + or not file_path.endswith(".properties") + or not basename_current_file.startswith("messages_") + ): + continue + + current_properties = parse_properties_file(branch + file_path) + updated_properties = [] + for ref_entry in reference_properties: + ref_entry_copy = copy.deepcopy(ref_entry) + for current_entry in current_properties: + if current_entry["type"] == "entry": + if ref_entry_copy["type"] != "entry": + continue + if ref_entry_copy["key"] == current_entry["key"]: + ref_entry_copy["value"] = current_entry["value"] + updated_properties.append(ref_entry_copy) + write_json_file(branch + file_path, updated_properties) + + +def check_for_missing_keys(reference_file, file_list, branch): + update_missing_keys(reference_file, file_list, branch + "/") def read_properties(file_path): @@ -7,87 +126,97 @@ def read_properties(file_path): return file.read().splitlines() -def check_difference(reference_file, file_list, branch): +def check_for_differences(reference_file, file_list, branch): reference_branch = reference_file.split("/")[0] basename_reference_file = os.path.basename(reference_file) report = [] report.append( - f"#### Checking with the file `{basename_reference_file}` from the `{reference_branch}` - Checking the `{branch}`" + f"### 📋 Checking with the file `{basename_reference_file}` from the `{reference_branch}` - Checking the `{branch}`" ) - reference_list = read_properties(reference_file) - is_diff = False + reference_lines = read_properties(reference_file) + has_differences = False + + only_reference_file = True for file_path in file_list: basename_current_file = os.path.basename(branch + "/" + file_path) if ( - branch + "/" + file_path == reference_file + basename_current_file == basename_reference_file or not file_path.endswith(".properties") or not basename_current_file.startswith("messages_") ): - # report.append(f"File '{basename_current_file}' is ignored.") continue - report.append(f"Checking the language file `{basename_current_file}`...") - current_list = read_properties(branch + "/" + file_path) - reference_list_len = len(reference_list) - current_list_len = len(current_list) + only_reference_file = False + report.append(f"#### 🗂️ **Checking File:** `{basename_current_file}`...") + current_lines = read_properties(branch + "/" + file_path) + reference_line_count = len(reference_lines) + current_line_count = len(current_lines) - if reference_list_len != current_list_len: + if reference_line_count != current_line_count: report.append("") - report.append("- ❌ Test 1 failed! Difference in the file!") - is_diff = True - if reference_list_len > current_list_len: + report.append("- **Test 1 Status:** ❌ Failed") + has_differences = True + if reference_line_count > current_line_count: report.append( - f" - Missing lines! Either comments, empty lines, or translation strings are missing! {reference_list_len}:{current_list_len}" + f" - **Issue:** Missing lines! Comments, empty lines, or translation strings are missing. Details: {reference_line_count} (reference) vs {current_line_count} (current)." ) - elif reference_list_len < current_list_len: + elif reference_line_count < current_line_count: report.append( - f" - Too many lines! Check your translation files! {reference_list_len}:{current_list_len}" + f" - **Issue:** Too many lines! Check your translation files! Details: {reference_line_count} (reference) vs {current_line_count} (current)." ) else: - report.append("- ✅ Test 1 passed") - if 1 == 1: - current_keys = [] - reference_keys = [] - for item in current_list: - if not item.startswith("#") and item != "" and "=" in item: - key, _ = item.split("=", 1) - current_keys.append(key) - for item in reference_list: - if not item.startswith("#") and item != "" and "=" in item: - key, _ = item.split("=", 1) - reference_keys.append(key) + report.append("- **Test 1 Status:** ✅ Passed") - current_set = set(current_keys) - reference_set = set(reference_keys) - set_test1 = current_set.difference(reference_set) - set_test2 = reference_set.difference(current_set) - set_test1_list = list(set_test1) - set_test2_list = list(set_test2) + # Check for missing or extra keys + current_keys = [] + reference_keys = [] + for line in current_lines: + if not line.startswith("#") and line != "" and "=" in line: + key, _ = line.split("=", 1) + current_keys.append(key) + for line in reference_lines: + if not line.startswith("#") and line != "" and "=" in line: + key, _ = line.split("=", 1) + reference_keys.append(key) - if len(set_test1_list) > 0 or len(set_test2_list) > 0: - is_diff = True - set_test1_list = "`, `".join(set_test1_list) - set_test2_list = "`, `".join(set_test2_list) - report.append("- ❌ Test 2 failed") - if len(set_test1_list) > 0: - report.append( - f" - There are keys in ***{basename_current_file}*** `{set_test1_list}` that are not present in ***{basename_reference_file}***!" - ) - if len(set_test2_list) > 0: - report.append( - f" - There are keys in ***{basename_reference_file}*** `{set_test2_list}` that are not present in ***{basename_current_file}***!" - ) - else: - report.append("- ✅ Test 2 passed") + current_keys_set = set(current_keys) + reference_keys_set = set(reference_keys) + missing_keys = current_keys_set.difference(reference_keys_set) + extra_keys = reference_keys_set.difference(current_keys_set) + missing_keys_list = list(missing_keys) + extra_keys_list = list(extra_keys) + + if missing_keys_list or extra_keys_list: + has_differences = True + missing_keys_str = "`, `".join(missing_keys_list) + extra_keys_str = "`, `".join(extra_keys_list) + report.append("- **Test 2 Status:** ❌ Failed") + if missing_keys_list: + report.append( + f" - **Issue:** There are keys in ***{basename_current_file}*** `{missing_keys_str}` that are not present in ***{basename_reference_file}***!" + ) + if extra_keys_list: + report.append( + f" - **Issue:** There are keys in ***{basename_reference_file}*** `{extra_keys_str}` that are not present in ***{basename_current_file}***!" + ) + else: + report.append("- **Test 2 Status:** ✅ Passed") + if has_differences: + report.append("") + report.append(f"#### 🚧 ***{basename_current_file}*** will be corrected...") + report.append("") + report.append("---") report.append("") - - report.append("") - if is_diff: - report.append("## ❌ Check fail") + update_file_list = glob.glob(branch + "/src/**/messages_*.properties", recursive=True) + update_missing_keys(reference_file, update_file_list) + if has_differences: + report.append("## ❌ Overall Check Status: **_Failed_**") else: - report.append("## ✅ Check success") - print("\n".join(report)) + report.append("## ✅ Overall Check Status: **_Success_**") + + if not only_reference_file: + print("\n".join(report)) if __name__ == "__main__": @@ -106,10 +235,16 @@ if __name__ == "__main__": parser.add_argument( "--files", nargs="+", - required=True, + required=False, help="List of changed files, separated by spaces.", ) args = parser.parse_args() file_list = args.files - check_difference(args.reference_file, file_list, args.branch) + if file_list is None: + file_list = glob.glob( + os.getcwd() + "/src/**/messages_*.properties", recursive=True + ) + update_missing_keys(args.reference_file, file_list) + else: + check_for_differences(args.reference_file, file_list, args.branch) diff --git a/.github/workflows/check_properties.yml b/.github/workflows/check_properties.yml index 544e8410..f5417465 100644 --- a/.github/workflows/check_properties.yml +++ b/.github/workflows/check_properties.yml @@ -1,15 +1,22 @@ -name: Check Properties Files in PR +name: Check Properties Files on: pull_request_target: types: [opened, synchronize, reopened] paths: - "src/main/resources/messages_*.properties" + push: + paths: + - "src/main/resources/messages_en_GB.properties" + +permissions: + contents: write + pull-requests: write jobs: check-files: + if: github.event_name == 'pull_request_target' runs-on: ubuntu-latest - steps: - name: Checkout PR branch uses: actions/checkout@v4 @@ -54,11 +61,12 @@ jobs: id: determine-file run: | echo "Determining reference file..." - if echo "${{ env.CHANGED_FILES }}"| grep -q 'src/main/resources/messages_en_GB.properties'; then + if echo "${{ env.CHANGED_FILES }}" | grep -q 'src/main/resources/messages_en_GB.properties'; then echo "REFERENCE_FILE=pr-branch/src/main/resources/messages_en_GB.properties" >> $GITHUB_ENV else echo "REFERENCE_FILE=main-branch/src/main/resources/messages_en_GB.properties" >> $GITHUB_ENV fi + echo "REFERENCE_FILE=${{ env.REFERENCE_FILE }}" - name: Show REFERENCE_FILE run: echo "Reference file is set to ${{ env.REFERENCE_FILE }}" @@ -71,26 +79,121 @@ jobs: - name: Capture output id: capture-output run: | - if [ -f failure.txt ]; then + if [ -f failure.txt ] && [ -s failure.txt ]; then echo "Test failed, capturing output..." - # Use the cat command to avoid issues with special characters in environment variables ERROR_OUTPUT=$(cat failure.txt) echo "ERROR_OUTPUT<> $GITHUB_ENV echo "$ERROR_OUTPUT" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV echo $ERROR_OUTPUT + else + echo "No errors found." + echo "ERROR_OUTPUT=" >> $GITHUB_ENV fi - name: Post comment on PR + if: env.ERROR_OUTPUT != '' uses: actions/github-script@v7 with: script: | const { GITHUB_REPOSITORY, ERROR_OUTPUT } = process.env; const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/'); - const prNumber = context.issue.number; // Pull request number from context - await github.rest.issues.createComment({ + const prNumber = context.issue.number; + + // Find existing comment + const comments = await github.rest.issues.listComments({ owner: repoOwner, repo: repoName, - issue_number: prNumber, - body: `## Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n` + issue_number: prNumber }); + + const comment = comments.data.find(c => c.body.includes("## 🚀 Translation Verification Summary")); + + // Only allow the action user to update comments + const expectedActor = "github-actions[bot]"; + + if (comment && comment.user.login === expectedActor) { + // Update existing comment + await github.rest.issues.updateComment({ + owner: repoOwner, + repo: repoName, + comment_id: comment.id, + body: `## 🚀 Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n` + }); + console.log("Updated existing comment."); + } else if (!comment) { + // Create new comment if no existing comment is found + await github.rest.issues.createComment({ + owner: repoOwner, + repo: repoName, + issue_number: prNumber, + body: `## 🚀 Translation Verification Summary\n\n\n${ERROR_OUTPUT}\n` + }); + console.log("Created new comment."); + } else { + console.log("Comment update attempt denied. Actor does not match."); + } + + - name: Set up git config + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add translation keys + run: | + cd ${{ env.BRANCH_PATH }} + git add src/main/resources/messages_*.properties + git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV + git commit -m "Update translation files" || echo "No changes to commit" + - name: Push + if: env.CHANGES_DETECTED == 'true' + run: | + cd pr-branch + git push origin ${{ github.head_ref }} || echo "Push failed: possibly no changes to push" + + update-translations-main: + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Run Python script to check files + id: run-check + run: | + python .github/scripts/check_language_properties.py --reference-file src/main/resources/messages_en_GB.properties --branch main + + - name: Set up git config + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Add translation keys + run: | + git add src/main/resources/messages_*.properties + git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV + + - name: Create Pull Request + id: cpr + if: env.CHANGES_DETECTED == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update translation files" + committer: GitHub Action + author: GitHub Action + signoff: true + branch: update_translation_files + title: "Update translation files" + body: | + Auto-generated by [create-pull-request][1] + + [1]: https://github.com/peter-evans/create-pull-request + labels: Translation + draft: false + delete-branch: true