From 2fa68be36b155a73e7a412255030bbece7d84ef2 Mon Sep 17 00:00:00 2001 From: Anthony Stirling <77850077+Frooodle@users.noreply.github.com> Date: Thu, 18 Jan 2024 21:57:41 +0000 Subject: [PATCH] pipeline fixes --- .../api/SplitPdfBySectionsController.java | 4 +- .../converters/ConvertImgPDFController.java | 2 +- .../api/converters/ConvertMarkdownToPdf.java | 2 +- .../api/misc/OverlayImageController.java | 2 +- .../api/pipeline/ApiDocService.java | 45 ++++++- .../api/pipeline/PipelineProcessor.java | 117 +++++++++++------- .../api/security/CertSignController.java | 2 +- .../software/SPDF/utils/FileToPdf.java | 5 +- .../resources/static/js/languageSelection.js | 17 ++- 9 files changed, 139 insertions(+), 57 deletions(-) diff --git a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java index de3e5a4b..90418169 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java @@ -121,8 +121,8 @@ public class SplitPdfBySectionsController { subDoc, subPage, AppendMode.APPEND, true, true)) { // Set clipping area and position float translateX = -subPageWidth * i; - - //float translateY = height - subPageHeight * (verticalDivisions - j); + + // float translateY = height - subPageHeight * (verticalDivisions - j); float translateY = -subPageHeight * (verticalDivisions - 1 - j); contentStream.saveGraphicsState(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertImgPDFController.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertImgPDFController.java index a3ea2841..a4f8a98d 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertImgPDFController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertImgPDFController.java @@ -96,7 +96,7 @@ public class ConvertImgPDFController { @Operation( summary = "Convert images to a PDF file", description = - "This endpoint converts one or more images to a PDF file. Users can specify whether to stretch the images to fit the PDF page, and whether to automatically rotate the images. Input:Image Output:PDF Type:SISO?") + "This endpoint converts one or more images to a PDF file. Users can specify whether to stretch the images to fit the PDF page, and whether to automatically rotate the images. Input:Image Output:PDF Type:MISO") public ResponseEntity convertToPdf(@ModelAttribute ConvertToPdfRequest request) throws IOException { MultipartFile[] file = request.getFileInput(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertMarkdownToPdf.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertMarkdownToPdf.java index fe590459..1b6efefb 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertMarkdownToPdf.java +++ b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertMarkdownToPdf.java @@ -39,7 +39,7 @@ public class ConvertMarkdownToPdf { @Operation( summary = "Convert a Markdown file to PDF", description = - "This endpoint takes a Markdown file input, converts it to HTML, and then to PDF format.") + "This endpoint takes a Markdown file input, converts it to HTML, and then to PDF format. Input:MARKDOWN Output:PDF Type:SISO") public ResponseEntity markdownToPdf(@ModelAttribute GeneralFile request) throws Exception { MultipartFile fileInput = request.getFileInput(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java b/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java index 9fe6249c..caf2efaf 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/misc/OverlayImageController.java @@ -30,7 +30,7 @@ public class OverlayImageController { @Operation( summary = "Overlay image onto a PDF file", description = - "This endpoint overlays an image onto a PDF file at the specified coordinates. The image can be overlaid on every page of the PDF if specified. Input:PDF/IMAGE Output:PDF Type:MF-SISO") + "This endpoint overlays an image onto a PDF file at the specified coordinates. The image can be overlaid on every page of the PDF if specified. Input:PDF/IMAGE Output:PDF Type:SISO") public ResponseEntity overlayImage(@ModelAttribute OverlayImageRequest request) { MultipartFile pdfFile = request.getFileInput(); MultipartFile imageFile = request.getImageFile(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java index 6ed5f51d..9df659ac 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java @@ -1,5 +1,6 @@ package stirling.software.SPDF.controller.api.pipeline; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -22,7 +23,7 @@ import jakarta.servlet.ServletContext; import stirling.software.SPDF.SPdfApplication; import stirling.software.SPDF.model.ApiEndpoint; import stirling.software.SPDF.model.Role; - +import java.util.List; @Service public class ApiDocService { @@ -38,6 +39,48 @@ public class ApiDocService { return "http://localhost:" + port + contextPath + "/v1/api-docs"; } + Map> outputToFileTypes = new HashMap<>(); + + public List getExtensionTypes(boolean output, String operationName) { + if(outputToFileTypes.size() == 0) { + outputToFileTypes.put("PDF", Arrays.asList("pdf")); + outputToFileTypes.put("IMAGE", Arrays.asList("png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff", "svg", "psd", "ai", "eps")); + outputToFileTypes.put("ZIP", Arrays.asList("zip", "rar", "7z", "tar", "gz", "bz2", "xz", "lz", "lzma", "z")); + outputToFileTypes.put("WORD", Arrays.asList("doc", "docx", "odt", "rtf")); + outputToFileTypes.put("CSV", Arrays.asList("csv")); + outputToFileTypes.put("JS", Arrays.asList("js", "jsx")); + outputToFileTypes.put("HTML", Arrays.asList("html", "htm", "xhtml")); + outputToFileTypes.put("JSON", Arrays.asList("json")); + outputToFileTypes.put("TXT", Arrays.asList("txt", "text", "md", "markdown")); + outputToFileTypes.put("PPT", Arrays.asList("ppt", "pptx", "odp")); + outputToFileTypes.put("XML", Arrays.asList("xml", "xsd", "xsl")); + outputToFileTypes.put("BOOK", Arrays.asList("epub", "mobi", "azw3", "fb2", "txt", "docx")); // As noted before, "Boolean" isn't a file type but a value type. + } + + if (apiDocsJsonRootNode == null || apiDocumentation.size() == 0) { + loadApiDocumentation(); + } + if (!apiDocumentation.containsKey(operationName)) { + return null; + } + + ApiEndpoint endpoint = apiDocumentation.get(operationName); + String description = endpoint.getDescription(); + Pattern pattern = null; + if(output) { + pattern = Pattern.compile("Output:(\\w+)"); + } else { + pattern = Pattern.compile("Input:(\\w+)"); + } + Matcher matcher = pattern.matcher(description); + while (matcher.find()) { + String type = matcher.group(1).toUpperCase(); + if(outputToFileTypes.containsKey(type)) { + return outputToFileTypes.get(type); + } + } + return null; + } @Autowired(required = false) private UserServiceInterface userService; diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java index 34dcb613..392bdac8 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -11,6 +11,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -82,15 +83,13 @@ public class PipelineProcessor { operation, isMultiInputOperation); Map parameters = pipelineOperation.getParameters(); - String inputFileExtension = ""; + List inputFileTypes = apiDocService.getExtensionTypes(false, operation); + if(inputFileTypes == null) { + inputFileTypes = new ArrayList(Arrays.asList("ALL")); + } + //List outputFileTypes = apiDocService.getExtensionTypes(true, operation); + - // TODO - // if (operationNode.has("inputFileType")) { - // inputFileExtension = operationNode.get("inputFileType").asText(); - // } else { - inputFileExtension = ".pdf"; - // } - final String finalInputFileExtension = inputFileExtension; String url = getBaseUrl() + operation; @@ -98,38 +97,40 @@ public class PipelineProcessor { if (!isMultiInputOperation) { for (Resource file : outputFiles) { boolean hasInputFileType = false; - if (file.getFilename().endsWith(inputFileExtension)) { - hasInputFileType = true; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("fileInput", file); - - for (Entry entry : parameters.entrySet()) { - body.add(entry.getKey(), entry.getValue()); - } - - ResponseEntity response = sendWebRequest(url, body); - - // If the operation is filter and the response body is null or empty, skip - // this - // file - if (operation.startsWith("filter-") - && (response.getBody() == null || response.getBody().length == 0)) { - logger.info("Skipping file due to failing {}", operation); - continue; - } - - if (!response.getStatusCode().equals(HttpStatus.OK)) { - logPrintStream.println("Error: " + response.getBody()); - hasErrors = true; - continue; - } - processOutputFiles(operation, file.getFilename(), response, newOutputFiles); + for (String extension : inputFileTypes) { + if (extension.equals("ALL") || file.getFilename().endsWith(extension)) { + hasInputFileType = true; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("fileInput", file); + + for (Entry entry : parameters.entrySet()) { + body.add(entry.getKey(), entry.getValue()); + } + + ResponseEntity response = sendWebRequest(url, body); + + // If the operation is filter and the response body is null or empty, skip + // this + // file + if (operation.startsWith("filter-") + && (response.getBody() == null || response.getBody().length == 0)) { + logger.info("Skipping file due to failing {}", operation); + continue; + } + + if (!response.getStatusCode().equals(HttpStatus.OK)) { + logPrintStream.println("Error: " + response.getBody()); + hasErrors = true; + continue; + } + processOutputFiles(operation, response, newOutputFiles); + } } if (!hasInputFileType) { logPrintStream.println( "No files with extension " - + inputFileExtension + + String.join(", ", inputFileTypes) + " found for operation " + operation); hasErrors = true; @@ -138,13 +139,16 @@ public class PipelineProcessor { } else { // Filter and collect all files that match the inputFileExtension - List matchingFiles = - outputFiles.stream() - .filter( - file -> - file.getFilename() - .endsWith(finalInputFileExtension)) - .collect(Collectors.toList()); + List matchingFiles; + if (inputFileTypes.contains("ALL")) { + matchingFiles = new ArrayList<>(outputFiles); + } else { + final List finalinputFileTypes = inputFileTypes; + matchingFiles = + outputFiles.stream() + .filter(file -> finalinputFileTypes.stream().anyMatch(file.getFilename()::endsWith)) + .collect(Collectors.toList()); + } // Check if there are matching files if (!matchingFiles.isEmpty()) { @@ -166,7 +170,6 @@ public class PipelineProcessor { if (response.getStatusCode().equals(HttpStatus.OK)) { processOutputFiles( operation, - matchingFiles.get(0).getFilename(), response, newOutputFiles); } else { @@ -178,7 +181,7 @@ public class PipelineProcessor { } else { logPrintStream.println( "No files with extension " - + inputFileExtension + + String.join(", ", inputFileTypes) + " found for multi-input operation " + operation); hasErrors = true; @@ -210,10 +213,30 @@ public class PipelineProcessor { // Make the request to the REST endpoint return restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); } + + public static String removeTrailingNaming(String filename) { + // Splitting filename into name and extension + int dotIndex = filename.lastIndexOf("."); + if (dotIndex == -1) { + // No extension found + return filename; + } + String name = filename.substring(0, dotIndex); + String extension = filename.substring(dotIndex); + + // Finding the last underscore + int underscoreIndex = name.lastIndexOf("_"); + if (underscoreIndex == -1) { + // No underscore found + return filename; + } + + // Removing the last part and reattaching the extension + return name.substring(0, underscoreIndex) + extension; + } private List processOutputFiles( String operation, - String fileName, ResponseEntity response, List newOutputFiles) throws IOException { @@ -227,9 +250,9 @@ public class PipelineProcessor { newFilename = extractFilename(response); } else { // Otherwise, keep the original filename. - newFilename = fileName; + newFilename = removeTrailingNaming(extractFilename(response)); } - + // Check if the response body is a zip file if (isZip(response.getBody())) { // Unzip the file and add all the files to the new output files diff --git a/src/main/java/stirling/software/SPDF/controller/api/security/CertSignController.java b/src/main/java/stirling/software/SPDF/controller/api/security/CertSignController.java index f7c7390e..e7cccb44 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/security/CertSignController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/security/CertSignController.java @@ -74,7 +74,7 @@ public class CertSignController { @Operation( summary = "Sign PDF with a Digital Certificate", description = - "This endpoint accepts a PDF file, a digital certificate and related information to sign the PDF. It then returns the digitally signed PDF file. Input:PDF Output:PDF Type:MF-SISO") + "This endpoint accepts a PDF file, a digital certificate and related information to sign the PDF. It then returns the digitally signed PDF file. Input:PDF Output:PDF Type:SISO") public ResponseEntity signPDFWithCert(@ModelAttribute SignPDFWithCertRequest request) throws Exception { MultipartFile pdf = request.getFileInput(); diff --git a/src/main/java/stirling/software/SPDF/utils/FileToPdf.java b/src/main/java/stirling/software/SPDF/utils/FileToPdf.java index ebdbf4fa..885cb6e5 100644 --- a/src/main/java/stirling/software/SPDF/utils/FileToPdf.java +++ b/src/main/java/stirling/software/SPDF/utils/FileToPdf.java @@ -36,6 +36,10 @@ public class FileToPdf { } else { command.add("wkhtmltopdf"); command.add("--enable-local-file-access"); + command.add("--load-error-handling"); + command.add("ignore"); + command.add("--load-media-error-handling"); + command.add("ignore"); } command.add(tempInputFile.toString()); @@ -130,7 +134,6 @@ public class FileToPdf { command.add("ebook-convert"); command.add(tempInputFile.toString()); command.add(tempOutputFile.toString()); - ProcessExecutorResult returnCode = ProcessExecutor.getInstance(ProcessExecutor.Processes.CALIBRE) .runCommandWithOutputHandling(command); diff --git a/src/main/resources/static/js/languageSelection.js b/src/main/resources/static/js/languageSelection.js index a5e7be2a..432b467e 100644 --- a/src/main/resources/static/js/languageSelection.js +++ b/src/main/resources/static/js/languageSelection.js @@ -1,7 +1,20 @@ document.addEventListener('DOMContentLoaded', function() { setLanguageForDropdown('.lang_dropdown-item'); - const defaultLocale = document.documentElement.lang || 'en_GB'; - const storedLocale = localStorage.getItem('languageCode') || defaultLocale; + + // Detect the browser's preferred language + let browserLang = navigator.language || navigator.userLanguage; + // Convert to a format consistent with your language codes (e.g., en-GB, fr-FR) + browserLang = browserLang.replace('-', '_'); + + // Check if the dropdown contains the browser's language + const dropdownLangExists = document.querySelector(`.lang_dropdown-item[data-language-code="${browserLang}"]`); + + // Set the default language to browser's language or 'en_GB' if not found in the dropdown + const defaultLocale = dropdownLangExists ? browserLang : 'en_GB'; + const storedLocale = localStorage.getItem('languageCode') || defaultLocale; + + + const dropdownItems = document.querySelectorAll('.lang_dropdown-item'); for (let i = 0; i < dropdownItems.length; i++) {