diff --git a/.gitignore b/.gitignore index 67a9a4b5..b0c87c9e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,8 @@ local.properties .classpath .project version.properties -pipeline/ - +pipeline/watchedFolders/ +pipeline/finishedFolders/ #### Stirling-PDF Files ### customFiles/ configs/ diff --git a/Dockerfile b/Dockerfile index 657db0b5..f0eb592c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ ARG VERSION_TAG # Set Environment Variables ENV DOCKER_ENABLE_SECURITY=false \ HOME=/home/stirlingpdfuser \ - VERSION_TAG=$VERSION_TAG + VERSION_TAG=$VERSION_TAG \ + JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMPercentage=75" # PUID=1000 \ # PGID=1000 \ # UMASK=022 \ @@ -18,13 +19,14 @@ ENV DOCKER_ENABLE_SECURITY=false \ ## mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME # Set up necessary directories and permissions -RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles +RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /logs /customFiles /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders ##&& \ ## chown -R stirlingpdfuser:stirlingpdfgroup /scripts /usr/share/fonts/opentype/noto /usr/share/tesseract-ocr /configs /customFiles && \ ## chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/tesseract-ocr-original # Copy necessary files COPY ./scripts/* /scripts/ +COPY ./pipeline/ /pipeline/ COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/ COPY build/libs/*.jar app.jar diff --git a/Dockerfile-lite b/Dockerfile-lite index 068e04ca..054c7639 100644 --- a/Dockerfile-lite +++ b/Dockerfile-lite @@ -17,7 +17,8 @@ RUN apt-get update && \ # Set Environment Variables ENV DOCKER_ENABLE_SECURITY=false \ HOME=/home/stirlingpdfuser \ - VERSION_TAG=$VERSION_TAG + VERSION_TAG=$VERSION_TAG \ + JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMPercentage=75" # PUID=1000 \ # PGID=1000 \ # UMASK=022 \ @@ -28,13 +29,14 @@ ENV DOCKER_ENABLE_SECURITY=false \ # mkdir -p $HOME && chown stirlingpdfuser:stirlingpdfgroup $HOME # Set up necessary directories and permissions -RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles +RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles /logs /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders # chown -R stirlingpdfuser:stirlingpdfgroup /usr/share/fonts/opentype/noto /configs /customFiles # Copy necessary files COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh +COPY ./pipeline/ /pipeline/ COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/ COPY build/libs/*.jar app.jar diff --git a/Dockerfile-ultra-lite b/Dockerfile-ultra-lite index 621ed58a..d77c1b94 100644 --- a/Dockerfile-ultra-lite +++ b/Dockerfile-ultra-lite @@ -6,7 +6,8 @@ ARG VERSION_TAG # Set Environment Variables ENV DOCKER_ENABLE_SECURITY=false \ HOME=/home/stirlingpdfuser \ - VERSION_TAG=$VERSION_TAG + VERSION_TAG=$VERSION_TAG \ + JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS -XX:MaxRAMPercentage=75" # PUID=1000 \ # PGID=1000 \ # UMASK=022 \ @@ -18,12 +19,12 @@ ENV DOCKER_ENABLE_SECURITY=false \ # Set up necessary directories and permissions #RUN mkdir -p /scripts /configs /customFiles && \ -# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles - +# chown -R stirlingpdfuser:stirlingpdfgroup /scripts /configs /customFiles /logs /pipeline /pipeline/defaultWebUIConfigs /pipeline/watchedFolders /pipeline/finishedFolders + RUN mkdir -p /scripts /usr/share/fonts/opentype/noto /configs /customFiles COPY ./scripts/download-security-jar.sh /scripts/download-security-jar.sh COPY ./scripts/init-without-ocr.sh /scripts/init-without-ocr.sh - +COPY ./pipeline/ /pipeline/ COPY build/libs/*.jar app.jar # Set font cache and permissions diff --git a/build.gradle b/build.gradle index 66de762e..dd854e45 100644 --- a/build.gradle +++ b/build.gradle @@ -8,7 +8,7 @@ plugins { } group = 'stirling.software' -version = '0.17.2' +version = '0.18.0' sourceCompatibility = '17' repositories { @@ -48,7 +48,7 @@ launch4j { errTitle="Encountered error, Do you have Java 17?" downloadUrl="https://download.oracle.com/java/17/latest/jdk-17_windows-x64_bin.exe" - variables=["BROWSER_OPEN=true"] + variables=["BROWSER_OPEN=true", "ENDPOINTS_GROUPS_TO_REMOVE=CLI"] jreMinVersion="17" mutexName="Stirling-PDF" @@ -68,17 +68,17 @@ dependencies { implementation 'org.springframework:spring-webmvc:6.0.15' implementation 'org.yaml:snakeyaml:2.1' - implementation 'org.springframework.boot:spring-boot-starter-web:3.1.6' - implementation 'org.springframework.boot:spring-boot-starter-thymeleaf:3.1.6' + implementation 'org.springframework.boot:spring-boot-starter-web:3.2.1' + implementation 'org.springframework.boot:spring-boot-starter-thymeleaf:3.2.1' if (System.getenv('DOCKER_ENABLE_SECURITY') != 'false') { - implementation 'org.springframework.boot:spring-boot-starter-security:3.1.6' + implementation 'org.springframework.boot:spring-boot-starter-security:3.2.1' implementation 'org.thymeleaf.extras:thymeleaf-extras-springsecurity5:3.1.2.RELEASE' implementation "org.springframework.boot:spring-boot-starter-data-jpa" implementation "com.h2database:h2" } - testImplementation 'org.springframework.boot:spring-boot-starter-test:3.1.6' + testImplementation 'org.springframework.boot:spring-boot-starter-test:3.2.1' // Batik implementation 'org.apache.xmlgraphics:batik-all:1.17' diff --git a/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json b/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json new file mode 100644 index 00000000..ba3228dc --- /dev/null +++ b/pipeline/defaultWebUIConfigs/Prepare-pdfs-for-email.json @@ -0,0 +1,39 @@ +{ + "name": "Prepare-pdfs-for-email", + "pipeline": [ + { + "operation": "/api/v1/misc/repair", + "parameters": {} + }, + { + "operation": "/api/v1/security/sanitize-pdf", + "parameters": { + "removeJavaScript": true, + "removeEmbeddedFiles": false, + "removeMetadata": false, + "removeLinks": false, + "removeFonts": false + } + }, + { + "operation": "/api/v1/misc/compress-pdf", + "parameters": { + "optimizeLevel": 2, + "expectedOutputSize": "" + } + }, + { + "operation": "/api/v1/general/split-by-size-or-count", + "parameters": { + "splitType": 0, + "splitValue": "15MB" + } + } + ], + "_examples": { + "outputDir": "{outputFolder}/{folderName}", + "outputFileName": "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir": "httpWebRequest", + "outputFileName": "{filename}" +} \ No newline at end of file diff --git a/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json b/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json new file mode 100644 index 00000000..3a989296 --- /dev/null +++ b/pipeline/defaultWebUIConfigs/split-rotate-auto-rename.json @@ -0,0 +1,33 @@ +{ + "name": "split-rotate-auto-rename", + "pipeline": [ + { + "operation": "/api/v1/general/split-pdf-by-sections", + "parameters": { + "horizontalDivisions": 2, + "verticalDivisions": 2, + "fileInput": "automated" + } + }, + { + "operation": "/api/v1/general/rotate-pdf", + "parameters": { + "angle": 90, + "fileInput": "automated" + } + }, + { + "operation": "/api/v1/misc/auto-rename", + "parameters": { + "useFirstTextAsFallback": false, + "fileInput": "automated" + } + } + ], + "_examples": { + "outputDir": "{outputFolder}/{folderName}", + "outputFileName": "{filename}-{pipelineName}-{date}-{time}" + }, + "outputDir": "{outputFolder}", + "outputFileName": "{filename}" +} \ No newline at end of file diff --git a/src/main/java/stirling/software/SPDF/SPdfApplication.java b/src/main/java/stirling/software/SPDF/SPdfApplication.java index 5dd7fed3..aab480ba 100644 --- a/src/main/java/stirling/software/SPDF/SPdfApplication.java +++ b/src/main/java/stirling/software/SPDF/SPdfApplication.java @@ -8,13 +8,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.core.env.Environment; +import org.springframework.scheduling.annotation.EnableScheduling; import jakarta.annotation.PostConstruct; import stirling.software.SPDF.config.ConfigInitializer; import stirling.software.SPDF.utils.GeneralUtils; @SpringBootApplication -//@EnableScheduling +@EnableScheduling public class SPdfApplication { @Autowired diff --git a/src/main/java/stirling/software/SPDF/config/AppConfig.java b/src/main/java/stirling/software/SPDF/config/AppConfig.java index 4d02f974..faf85b28 100644 --- a/src/main/java/stirling/software/SPDF/config/AppConfig.java +++ b/src/main/java/stirling/software/SPDF/config/AppConfig.java @@ -41,12 +41,16 @@ public class AppConfig { return (defaultNavBar != null) ? defaultNavBar : "Stirling PDF"; } + @Bean(name = "enableAlphaFunctionality") + public boolean enableAlphaFunctionality() { + return applicationProperties.getSystem().getEnableAlphaFunctionality() != null ? applicationProperties.getSystem().getEnableAlphaFunctionality() : false; + } + @Bean(name = "rateLimit") public boolean rateLimit() { String appName = System.getProperty("rateLimit"); if (appName == null) appName = System.getenv("rateLimit"); - System.out.println("rateLimit=" + appName); return (appName != null) ? Boolean.valueOf(appName) : false; } diff --git a/src/main/java/stirling/software/SPDF/config/MetricsFilter.java b/src/main/java/stirling/software/SPDF/config/MetricsFilter.java index 6ee59db7..9abb68bf 100644 --- a/src/main/java/stirling/software/SPDF/config/MetricsFilter.java +++ b/src/main/java/stirling/software/SPDF/config/MetricsFilter.java @@ -16,33 +16,35 @@ import jakarta.servlet.http.HttpServletResponse; @Component public class MetricsFilter extends OncePerRequestFilter { - private final MeterRegistry meterRegistry; + private final MeterRegistry meterRegistry; - @Autowired - public MetricsFilter(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - } + @Autowired + public MetricsFilter(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - @Override - protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) - throws ServletException, IOException { - String uri = request.getRequestURI(); - - //System.out.println("uri="+uri + ", method=" + request.getMethod() ); - // Ignore static resources - if (!(uri.startsWith("/js") || uri.startsWith("api-docs") || uri.endsWith("robots.txt") || uri.startsWith("/images") || uri.endsWith(".png") || uri.endsWith(".ico") || uri.endsWith(".css") || uri.endsWith(".svg")|| uri.endsWith(".js") || uri.contains("swagger") || uri.startsWith("/api"))) { - Counter counter = Counter.builder("http.requests") - .tag("uri", uri) - .tag("method", request.getMethod()) - .register(meterRegistry); + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + String uri = request.getRequestURI(); - counter.increment(); - //System.out.println("Counted"); - } + // System.out.println("uri="+uri + ", method=" + request.getMethod() ); + // Ignore static resources + if (!(uri.startsWith("/js") || uri.startsWith("/v1/api-docs") || uri.endsWith("robots.txt") + || uri.startsWith("/images") || uri.startsWith("/images")|| uri.endsWith(".png") || uri.endsWith(".ico") || uri.endsWith(".css") || uri.endsWith(".map") + || uri.endsWith(".svg") || uri.endsWith(".js") || uri.contains("swagger") + || uri.startsWith("/api/v1/info") || uri.startsWith("/site.webmanifest") || uri.startsWith("/fonts") || uri.startsWith("/pdfjs") )) { + + + + Counter counter = Counter.builder("http.requests").tag("uri", uri).tag("method", request.getMethod()) + .register(meterRegistry); - filterChain.doFilter(request, response); - } + counter.increment(); + // System.out.println("Counted"); + } - + filterChain.doFilter(request, response); + } } diff --git a/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java b/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java index 2583277e..5dba40d0 100644 --- a/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java +++ b/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java @@ -1,27 +1,42 @@ package stirling.software.SPDF.config; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import io.swagger.v3.oas.models.Components; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.security.SecurityRequirement; +import io.swagger.v3.oas.models.security.SecurityScheme; +import stirling.software.SPDF.model.ApplicationProperties; @Configuration public class OpenApiConfig { + @Autowired + ApplicationProperties applicationProperties; + @Bean public OpenAPI customOpenAPI() { - String version = getClass().getPackage().getImplementationVersion(); - if (version == null) { - - version = "1.0.0"; // default version if all else fails - - } + String version = getClass().getPackage().getImplementationVersion(); + if (version == null) { + version = "1.0.0"; // default version if all else fails + } + + SecurityScheme apiKeyScheme = new SecurityScheme().type(SecurityScheme.Type.APIKEY).in(SecurityScheme.In.HEADER) + .name("X-API-KEY"); + if (!applicationProperties.getSecurity().getEnableLogin()) { + return new OpenAPI().components(new Components()) + .info(new Info().title("Stirling PDF API").version(version).description( + "API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here.")); + } else { + return new OpenAPI().components(new Components().addSecuritySchemes("apiKey", apiKeyScheme)) + .info(new Info().title("Stirling PDF API").version(version).description( + "API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here.")) + .addSecurityItem(new SecurityRequirement().addList("apiKey")); + } - return new OpenAPI().components(new Components()).info( - new Info().title("Stirling PDF API").version(version).description("API documentation for all Server-Side processing.\nPlease note some functionality might be UI only and missing from here.")); } - -} +} \ No newline at end of file diff --git a/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationFailureHandler.java b/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationFailureHandler.java index f286f149..397a8a70 100644 --- a/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationFailureHandler.java +++ b/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationFailureHandler.java @@ -2,27 +2,46 @@ package stirling.software.SPDF.config.security; import java.io.IOException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.authentication.LockedException; import org.springframework.security.core.AuthenticationException; import org.springframework.security.web.authentication.SimpleUrlAuthenticationFailureHandler; +import org.springframework.stereotype.Component; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; - +@Component public class CustomAuthenticationFailureHandler extends SimpleUrlAuthenticationFailureHandler { + + @Autowired + private final LoginAttemptService loginAttemptService; + @Autowired + public CustomAuthenticationFailureHandler(LoginAttemptService loginAttemptService) { + this.loginAttemptService = loginAttemptService; + } + @Override public void onAuthenticationFailure(HttpServletRequest request, HttpServletResponse response, AuthenticationException exception) throws IOException, ServletException { String ip = request.getRemoteAddr(); logger.error("Failed login attempt from IP: " + ip); - if (exception.getClass().isAssignableFrom(BadCredentialsException.class)) { - setDefaultFailureUrl("/login?error=badcredentials"); - } else if (exception.getClass().isAssignableFrom(LockedException.class)) { + + String username = request.getParameter("username"); + if(loginAttemptService.loginAttemptCheck(username)) { setDefaultFailureUrl("/login?error=locked"); + + } else { + if (exception.getClass().isAssignableFrom(BadCredentialsException.class)) { + setDefaultFailureUrl("/login?error=badcredentials"); + } else if (exception.getClass().isAssignableFrom(LockedException.class)) { + setDefaultFailureUrl("/login?error=locked"); + } } + + super.onAuthenticationFailure(request, response, exception); } } diff --git a/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationSuccessHandler.java b/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationSuccessHandler.java new file mode 100644 index 00000000..cd2217e1 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/config/security/CustomAuthenticationSuccessHandler.java @@ -0,0 +1,44 @@ +package stirling.software.SPDF.config.security; + +import java.io.IOException; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.Authentication; +import org.springframework.security.web.authentication.SavedRequestAwareAuthenticationSuccessHandler; +import org.springframework.security.web.savedrequest.SavedRequest; +import org.springframework.stereotype.Component; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import stirling.software.SPDF.utils.RequestUriUtils; + +@Component +public class CustomAuthenticationSuccessHandler extends SavedRequestAwareAuthenticationSuccessHandler { + + @Autowired + private LoginAttemptService loginAttemptService; + + @Override + public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws ServletException, IOException { + String username = request.getParameter("username"); + loginAttemptService.loginSucceeded(username); + + + // Get the saved request + HttpSession session = request.getSession(false); + SavedRequest savedRequest = session != null ? (SavedRequest) session.getAttribute("SPRING_SECURITY_SAVED_REQUEST") : null; + if (savedRequest != null && !RequestUriUtils.isStaticResource(savedRequest.getRedirectUrl())) { + // Redirect to the original destination + super.onAuthenticationSuccess(request, response, authentication); + } else { + // Redirect to the root URL (considering context path) + getRedirectStrategy().sendRedirect(request, response, "/"); + } + + //super.onAuthenticationSuccess(request, response, authentication); + } + + +} diff --git a/src/main/java/stirling/software/SPDF/config/security/CustomUserDetailsService.java b/src/main/java/stirling/software/SPDF/config/security/CustomUserDetailsService.java index 7ae1680b..77db2cd4 100644 --- a/src/main/java/stirling/software/SPDF/config/security/CustomUserDetailsService.java +++ b/src/main/java/stirling/software/SPDF/config/security/CustomUserDetailsService.java @@ -5,6 +5,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.LockedException; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.userdetails.UserDetails; @@ -22,12 +23,18 @@ public class CustomUserDetailsService implements UserDetailsService { @Autowired private UserRepository userRepository; - + @Autowired + private LoginAttemptService loginAttemptService; + @Override public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { User user = userRepository.findByUsername(username) .orElseThrow(() -> new UsernameNotFoundException("No user found with username: " + username)); + if (loginAttemptService.isBlocked(username)) { + throw new LockedException("Your account has been locked due to too many failed login attempts."); + } + return new org.springframework.security.core.userdetails.User( user.getUsername(), user.getPassword(), diff --git a/src/main/java/stirling/software/SPDF/config/security/FirstLoginFilter.java b/src/main/java/stirling/software/SPDF/config/security/FirstLoginFilter.java index 8e612464..65acf148 100644 --- a/src/main/java/stirling/software/SPDF/config/security/FirstLoginFilter.java +++ b/src/main/java/stirling/software/SPDF/config/security/FirstLoginFilter.java @@ -15,6 +15,7 @@ import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import stirling.software.SPDF.model.User; +import stirling.software.SPDF.utils.RequestUriUtils; @Component public class FirstLoginFilter extends OncePerRequestFilter { @@ -28,11 +29,7 @@ public class FirstLoginFilter extends OncePerRequestFilter { String method = request.getMethod(); String requestURI = request.getRequestURI(); // Check if the request is for static resources - boolean isStaticResource = requestURI.startsWith("/css/") - || requestURI.startsWith("/js/") - || requestURI.startsWith("/images/") - || requestURI.startsWith("/public/") - || requestURI.endsWith(".svg"); + boolean isStaticResource = RequestUriUtils.isStaticResource(requestURI); // If it's a static resource, just continue the filter chain and skip the logic below if (isStaticResource) { diff --git a/src/main/java/stirling/software/SPDF/config/security/IPRateLimitingFilter.java b/src/main/java/stirling/software/SPDF/config/security/IPRateLimitingFilter.java new file mode 100644 index 00000000..03e34b57 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/config/security/IPRateLimitingFilter.java @@ -0,0 +1,65 @@ +package stirling.software.SPDF.config.security; +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import jakarta.servlet.Filter; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import stirling.software.SPDF.utils.RequestUriUtils; + +public class IPRateLimitingFilter implements Filter { + + private final ConcurrentHashMap requestCounts = new ConcurrentHashMap<>(); + private final ConcurrentHashMap getCounts = new ConcurrentHashMap<>(); + private final int maxRequests; + private final int maxGetRequests; + + public IPRateLimitingFilter(int maxRequests, int maxGetRequests) { + this.maxRequests = maxRequests; + this.maxGetRequests = maxGetRequests; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + if (request instanceof HttpServletRequest) { + HttpServletRequest httpRequest = (HttpServletRequest) request; + String method = httpRequest.getMethod(); + String requestURI = httpRequest.getRequestURI(); + // Check if the request is for static resources + boolean isStaticResource = RequestUriUtils.isStaticResource(requestURI); + + // If it's a static resource, just continue the filter chain and skip the logic below + if (isStaticResource) { + chain.doFilter(request, response); + return; + } + + String clientIp = request.getRemoteAddr(); + requestCounts.computeIfAbsent(clientIp, k -> new AtomicInteger(0)); + if (!"GET".equalsIgnoreCase(method)) { + + if (requestCounts.get(clientIp).incrementAndGet() > maxRequests) { + // Handle limit exceeded (e.g., send error response) + response.getWriter().write("Rate limit exceeded"); + return; + } + } else { + if (requestCounts.get(clientIp).incrementAndGet() > maxGetRequests) { + // Handle limit exceeded (e.g., send error response) + response.getWriter().write("GET Rate limit exceeded"); + return; + } + } + } + chain.doFilter(request, response); + } + + public void resetRequestCounts() { + requestCounts.clear(); + getCounts.clear(); + } +} diff --git a/src/main/java/stirling/software/SPDF/config/security/InitialSecuritySetup.java b/src/main/java/stirling/software/SPDF/config/security/InitialSecuritySetup.java index 842375d8..5d100dd8 100644 --- a/src/main/java/stirling/software/SPDF/config/security/InitialSecuritySetup.java +++ b/src/main/java/stirling/software/SPDF/config/security/InitialSecuritySetup.java @@ -37,8 +37,10 @@ public class InitialSecuritySetup { initialPassword = "stirling"; userService.saveUser(initialUsername, initialPassword, Role.ADMIN.getRoleId(), true); } - - + } + if(!userService.usernameExists(Role.INTERNAL_API_USER.getRoleId())) { + userService.saveUser(Role.INTERNAL_API_USER.getRoleId(), UUID.randomUUID().toString(), Role.INTERNAL_API_USER.getRoleId()); + userService.addApiKeyToUser(Role.INTERNAL_API_USER.getRoleId()); } } diff --git a/src/main/java/stirling/software/SPDF/config/security/LoginAttemptService.java b/src/main/java/stirling/software/SPDF/config/security/LoginAttemptService.java new file mode 100644 index 00000000..55a84449 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/config/security/LoginAttemptService.java @@ -0,0 +1,56 @@ +package stirling.software.SPDF.config.security; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PostConstruct; +import stirling.software.SPDF.model.ApplicationProperties; +import stirling.software.SPDF.model.AttemptCounter; + +@Service +public class LoginAttemptService { + + + @Autowired + ApplicationProperties applicationProperties; + + private int MAX_ATTEMPTS; + private long ATTEMPT_INCREMENT_TIME; + + + @PostConstruct + public void init() { + MAX_ATTEMPTS = applicationProperties.getSecurity().getLoginAttemptCount(); + ATTEMPT_INCREMENT_TIME = TimeUnit.MINUTES.toMillis(applicationProperties.getSecurity().getLoginResetTimeMinutes()); + } + + private final ConcurrentHashMap attemptsCache = new ConcurrentHashMap<>(); + + public void loginSucceeded(String key) { + attemptsCache.remove(key); + } + + public boolean loginAttemptCheck(String key) { + attemptsCache.compute(key, (k, attemptCounter) -> { + if (attemptCounter == null || attemptCounter.shouldReset(ATTEMPT_INCREMENT_TIME)) { + return new AttemptCounter(); + } else { + attemptCounter.increment(); + return attemptCounter; + } + }); + return attemptsCache.get(key).getAttemptCount() >= MAX_ATTEMPTS; + } + + + public boolean isBlocked(String key) { + AttemptCounter attemptCounter = attemptsCache.get(key); + if (attemptCounter != null) { + return attemptCounter.getAttemptCount() >= MAX_ATTEMPTS; + } + return false; + } + +} diff --git a/src/main/java/stirling/software/SPDF/config/security/RateLimitResetScheduler.java b/src/main/java/stirling/software/SPDF/config/security/RateLimitResetScheduler.java new file mode 100644 index 00000000..3ef8ef31 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/config/security/RateLimitResetScheduler.java @@ -0,0 +1,18 @@ +package stirling.software.SPDF.config.security; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +public class RateLimitResetScheduler { + + private final IPRateLimitingFilter rateLimitingFilter; + + public RateLimitResetScheduler(IPRateLimitingFilter rateLimitingFilter) { + this.rateLimitingFilter = rateLimitingFilter; + } + + @Scheduled(cron = "0 0 0 * * MON") // At 00:00 every Monday TODO: configurable + public void resetRateLimit() { + rateLimitingFilter.resetRequestCounts(); + } +} diff --git a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java index dfe782ac..e0b439db 100644 --- a/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java +++ b/src/main/java/stirling/software/SPDF/config/security/SecurityConfiguration.java @@ -6,7 +6,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; import org.springframework.security.authentication.dao.DaoAuthenticationProvider; -import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.method.configuration.EnableMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.core.userdetails.UserDetailsService; @@ -15,12 +15,13 @@ import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.authentication.rememberme.PersistentTokenRepository; +import org.springframework.security.web.savedrequest.NullRequestCache; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import stirling.software.SPDF.repository.JPATokenRepositoryImpl; @Configuration @EnableWebSecurity() -@EnableGlobalMethodSecurity(prePostEnabled = true) +@EnableMethodSecurity public class SecurityConfiguration { @Autowired @@ -40,6 +41,11 @@ public class SecurityConfiguration { @Autowired private UserAuthenticationFilter userAuthenticationFilter; + + + + @Autowired + private LoginAttemptService loginAttemptService; @Autowired private FirstLoginFilter firstLoginFilter; @@ -51,14 +57,18 @@ public class SecurityConfiguration { if(loginEnabledValue) { http.csrf(csrf -> csrf.disable()); + http.addFilterBefore(rateLimitingFilter(), UsernamePasswordAuthenticationFilter.class); http.addFilterAfter(firstLoginFilter, UsernamePasswordAuthenticationFilter.class); http .formLogin(formLogin -> formLogin .loginPage("/login") + .successHandler(new CustomAuthenticationSuccessHandler()) .defaultSuccessUrl("/") - .failureHandler(new CustomAuthenticationFailureHandler()) + .failureHandler(new CustomAuthenticationFailureHandler(loginAttemptService)) .permitAll() - ) + ).requestCache(requestCache -> requestCache + .requestCache(new NullRequestCache()) + ) .logout(logout -> logout .logoutRequestMatcher(new AntPathRequestMatcher("/logout")) .logoutSuccessUrl("/login?logout=true") @@ -70,8 +80,19 @@ public class SecurityConfiguration { .tokenValiditySeconds(1209600) // 2 weeks ) .authorizeHttpRequests(authz -> authz - .requestMatchers(req -> req.getRequestURI().startsWith("/login") || req.getRequestURI().endsWith(".svg") || req.getRequestURI().startsWith("/register") || req.getRequestURI().startsWith("/error") || req.getRequestURI().startsWith("/images/") || req.getRequestURI().startsWith("/public/") || req.getRequestURI().startsWith("/css/") || req.getRequestURI().startsWith("/js/")) - .permitAll() + .requestMatchers(req -> { + String uri = req.getRequestURI(); + String contextPath = req.getContextPath(); + + // Remove the context path from the URI + String trimmedUri = uri.startsWith(contextPath) ? uri.substring(contextPath.length()) : uri; + + return trimmedUri.startsWith("/login") || trimmedUri.endsWith(".svg") || + trimmedUri.startsWith("/register") || trimmedUri.startsWith("/error") || + trimmedUri.startsWith("/images/") || trimmedUri.startsWith("/public/") || + trimmedUri.startsWith("/css/") || trimmedUri.startsWith("/js/"); + } + ).permitAll() .anyRequest().authenticated() ) .userDetailsService(userDetailsService) @@ -84,8 +105,17 @@ public class SecurityConfiguration { } return http.build(); } + + + + @Bean + public IPRateLimitingFilter rateLimitingFilter() { + int maxRequestsPerIp = 1000000; // Example limit TODO add config level + return new IPRateLimitingFilter(maxRequestsPerIp, maxRequestsPerIp); + } + @Bean public DaoAuthenticationProvider authenticationProvider() { diff --git a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java index eca7f70e..ce77e5a4 100644 --- a/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java +++ b/src/main/java/stirling/software/SPDF/config/security/UserAuthenticationFilter.java @@ -74,15 +74,17 @@ public class UserAuthenticationFilter extends OncePerRequestFilter { // If we still don't have any authentication, deny the request if (authentication == null || !authentication.isAuthenticated()) { String method = request.getMethod(); - if ("GET".equalsIgnoreCase(method) && !"/login".equals(requestURI)) { - response.sendRedirect("/login"); // redirect to the login page + String contextPath = request.getContextPath(); + + if ("GET".equalsIgnoreCase(method) && ! (contextPath + "/login").equals(requestURI)) { + response.sendRedirect(contextPath + "/login"); // redirect to the login page return; } else { response.setStatus(HttpStatus.UNAUTHORIZED.value()); response.getWriter().write("Authentication required. Please provide a X-API-KEY in request header.\nThis is found in Settings -> Account Settings -> API Key\nAlternativly you can disable authentication if this is unexpected"); return; } - } + } filterChain.doFilter(request, response); } @@ -90,15 +92,17 @@ public class UserAuthenticationFilter extends OncePerRequestFilter { @Override protected boolean shouldNotFilter(HttpServletRequest request) throws ServletException { String uri = request.getRequestURI(); - + String contextPath = request.getContextPath(); String[] permitAllPatterns = { - "/login", - "/register", - "/error", - "/images/", - "/public/", - "/css/", - "/js/" + contextPath + "/login", + contextPath + "/register", + contextPath + "/error", + contextPath + "/images/", + contextPath + "/public/", + contextPath + "/css/", + contextPath + "/js/", + contextPath + "/pdfjs/", + contextPath + "/site.webmanifest" }; for (String pattern : permitAllPatterns) { diff --git a/src/main/java/stirling/software/SPDF/config/security/UserService.java b/src/main/java/stirling/software/SPDF/config/security/UserService.java index 46c5aeff..45794d92 100644 --- a/src/main/java/stirling/software/SPDF/config/security/UserService.java +++ b/src/main/java/stirling/software/SPDF/config/security/UserService.java @@ -16,11 +16,13 @@ import org.springframework.security.core.userdetails.UsernameNotFoundException; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.stereotype.Service; +import stirling.software.SPDF.controller.api.pipeline.UserServiceInterface; import stirling.software.SPDF.model.Authority; +import stirling.software.SPDF.model.Role; import stirling.software.SPDF.model.User; import stirling.software.SPDF.repository.UserRepository; @Service -public class UserService { +public class UserService implements UserServiceInterface{ @Autowired private UserRepository userRepository; @@ -136,6 +138,11 @@ public class UserService { public void deleteUser(String username) { Optional userOpt = userRepository.findByUsername(username); if (userOpt.isPresent()) { + for (Authority authority : userOpt.get().getAuthorities()) { + if (authority.getAuthority().equals(Role.INTERNAL_API_USER.getRoleId())) { + return; + } + } userRepository.delete(userOpt.get()); } } diff --git a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java index 29a2f426..f36f64da 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySectionsController.java @@ -29,12 +29,12 @@ import stirling.software.SPDF.model.api.SplitPdfBySectionsRequest; import stirling.software.SPDF.utils.WebResponseUtils; @RestController @RequestMapping("/api/v1/general") -@Tag(name = "Misc", description = "Miscellaneous APIs") +@Tag(name = "General", description = "General APIs") public class SplitPdfBySectionsController { @PostMapping(value = "/split-pdf-by-sections", consumes = "multipart/form-data") - @Operation(summary = "Split PDF pages into smaller sections", description = "Split each page of a PDF into smaller sections based on the user's choice (halves, thirds, quarters, etc.), both vertically and horizontally. Input: PDF, Split Parameters. Output: ZIP containing split documents.") + @Operation(summary = "Split PDF pages into smaller sections", description = "Split each page of a PDF into smaller sections based on the user's choice (halves, thirds, quarters, etc.), both vertically and horizontally. Input:PDF Output:ZIP-PDF Type:SISO") public ResponseEntity splitPdf(@ModelAttribute SplitPdfBySectionsRequest request) throws Exception { List splitDocumentsBoas = new ArrayList<>(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySizeController.java b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySizeController.java index 83e7b723..9b25e8be 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySizeController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/SplitPdfBySizeController.java @@ -26,13 +26,13 @@ import stirling.software.SPDF.utils.WebResponseUtils; @RestController @RequestMapping("/api/v1/general") -@Tag(name = "Misc", description = "Miscellaneous APIs") +@Tag(name = "General", description = "General APIs") public class SplitPdfBySizeController { @PostMapping(value = "/split-by-size-or-count", consumes = "multipart/form-data") @Operation(summary = "Auto split PDF pages into separate documents based on size or count", description = "split PDF into multiple paged documents based on size/count, ie if 20 pages and split into 5, it does 5 documents each 4 pages\r\n" - + " if 10MB and each page is 1MB and you enter 2MB then 5 docs each 2MB (rounded so that it accepts 1.9MB but not 2.1MB) Input:PDF Output:ZIP Type:SIMO") + + " if 10MB and each page is 1MB and you enter 2MB then 5 docs each 2MB (rounded so that it accepts 1.9MB but not 2.1MB) Input:PDF Output:ZIP-PDF Type:SISO") public ResponseEntity autoSplitPdf(@ModelAttribute SplitPdfBySizeOrCountRequest request) throws Exception { List splitDocumentsBoas = new ArrayList(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/UserController.java b/src/main/java/stirling/software/SPDF/controller/api/UserController.java index bf451567..01a50a3b 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/UserController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/UserController.java @@ -23,6 +23,7 @@ import org.springframework.web.servlet.view.RedirectView; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import stirling.software.SPDF.config.security.UserService; +import stirling.software.SPDF.model.Role; import stirling.software.SPDF.model.User; @Controller @@ -32,6 +33,7 @@ public class UserController { @Autowired private UserService userService; + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/register") public String register(@RequestParam String username, @RequestParam String password, Model model) { if(userService.usernameExists(username)) { @@ -43,6 +45,7 @@ public class UserController { return "redirect:/login?registered=true"; } + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/change-username-and-password") public RedirectView changeUsernameAndPassword(Principal principal, @RequestParam String currentPassword, @@ -85,7 +88,7 @@ public class UserController { } - + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/change-username") public RedirectView changeUsername(Principal principal, @RequestParam String currentPassword, @@ -122,7 +125,8 @@ public class UserController { return new RedirectView("/login?messageType=credsUpdated"); } - + + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/change-password") public RedirectView changePassword(Principal principal, @RequestParam String currentPassword, @@ -154,7 +158,7 @@ public class UserController { return new RedirectView("/login?messageType=credsUpdated"); } - + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/updateUserSettings") public String updateUserSettings(HttpServletRequest request, Principal principal) { Map paramMap = request.getParameterMap(); @@ -182,6 +186,18 @@ public class UserController { if(userService.usernameExists(username)) { return new RedirectView("/addUsers?messageType=usernameExists"); } + try { + // Validate the role + Role roleEnum = Role.fromString(role); + if (roleEnum == Role.INTERNAL_API_USER) { + // If the role is INTERNAL_API_USER, reject the request + return new RedirectView("/addUsers?messageType=invalidRole"); + } + } catch (IllegalArgumentException e) { + // If the role ID is not valid, redirect with an error message + return new RedirectView("/addUsers?messageType=invalidRole"); + } + userService.saveUser(username, password, role, forceChange); return new RedirectView("/addUsers"); // Redirect to account page after adding the user } @@ -203,6 +219,7 @@ public class UserController { return "redirect:/addUsers"; } + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/get-api-key") public ResponseEntity getApiKey(Principal principal) { if (principal == null) { @@ -216,6 +233,7 @@ public class UserController { return ResponseEntity.ok(apiKey); } + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @PostMapping("/update-api-key") public ResponseEntity updateApiKey(Principal principal) { if (principal == null) { diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertEpubToPdf.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertEpubToPdf.java deleted file mode 100644 index e821a36a..00000000 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertEpubToPdf.java +++ /dev/null @@ -1,130 +0,0 @@ -package stirling.software.SPDF.controller.api.converters; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ModelAttribute; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - -import io.swagger.v3.oas.annotations.Hidden; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.tags.Tag; -import stirling.software.SPDF.model.api.GeneralFile; -import stirling.software.SPDF.utils.FileToPdf; -import stirling.software.SPDF.utils.WebResponseUtils; - -@RestController -@RequestMapping("/api/v1/convert") -@Tag(name = "Convert", description = "Convert APIs") -public class ConvertEpubToPdf { - //TODO - @PostMapping(consumes = "multipart/form-data", value = "/epub-to-single-pdf") - @Hidden - @Operation( - summary = "Convert an EPUB file to a single PDF", - description = "This endpoint takes an EPUB file input and converts it to a single PDF." - ) - public ResponseEntity epubToSinglePdf( - @ModelAttribute GeneralFile request) - throws Exception { - MultipartFile fileInput = request.getFileInput(); - if (fileInput == null) { - throw new IllegalArgumentException("Please provide an EPUB file for conversion."); - } - - String originalFilename = fileInput.getOriginalFilename(); - if (originalFilename == null || !originalFilename.endsWith(".epub")) { - throw new IllegalArgumentException("File must be in .epub format."); - } - - Map epubContents = extractEpubContent(fileInput); - List htmlFilesOrder = getHtmlFilesOrderFromOpf(epubContents); - - List individualPdfs = new ArrayList<>(); - - for (String htmlFile : htmlFilesOrder) { - byte[] htmlContent = epubContents.get(htmlFile); - byte[] pdfBytes = FileToPdf.convertHtmlToPdf(htmlContent, htmlFile.replace(".html", ".pdf")); - individualPdfs.add(pdfBytes); - } - - // Pseudo-code to merge individual PDFs into one. - byte[] mergedPdfBytes = mergeMultiplePdfsIntoOne(individualPdfs); - - return WebResponseUtils.bytesToWebResponse(mergedPdfBytes, originalFilename.replace(".epub", ".pdf")); - } - - // Assuming a pseudo-code function that merges multiple PDFs into one. - private byte[] mergeMultiplePdfsIntoOne(List individualPdfs) { - // You can use a library such as PDFBox to perform the merging here. - // Return the byte[] of the merged PDF. - return null; - } - - private Map extractEpubContent(MultipartFile fileInput) throws IOException { - Map contentMap = new HashMap<>(); - - try (ZipInputStream zis = new ZipInputStream(fileInput.getInputStream())) { - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int read = 0; - while ((read = zis.read(buffer)) != -1) { - baos.write(buffer, 0, read); - } - contentMap.put(zipEntry.getName(), baos.toByteArray()); - zipEntry = zis.getNextEntry(); - } - } - - return contentMap; - } - - private List getHtmlFilesOrderFromOpf(Map epubContents) throws Exception { - String opfContent = new String(epubContents.get("OEBPS/content.opf")); // Adjusting for given path - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - InputSource is = new InputSource(new StringReader(opfContent)); - Document doc = dBuilder.parse(is); - - NodeList itemRefs = doc.getElementsByTagName("itemref"); - List htmlFilesOrder = new ArrayList<>(); - - for (int i = 0; i < itemRefs.getLength(); i++) { - Element itemRef = (Element) itemRefs.item(i); - String idref = itemRef.getAttribute("idref"); - - NodeList items = doc.getElementsByTagName("item"); - for (int j = 0; j < items.getLength(); j++) { - Element item = (Element) items.item(j); - if (idref.equals(item.getAttribute("id"))) { - htmlFilesOrder.add(item.getAttribute("href")); // Fetching the actual href - break; - } - } - } - - return htmlFilesOrder; - } - - -} diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java index 7c81edf2..d3f5c307 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java +++ b/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertWebsiteToPDF.java @@ -28,7 +28,7 @@ public class ConvertWebsiteToPDF { @PostMapping(consumes = "multipart/form-data", value = "/url/pdf") @Operation( summary = "Convert a URL to a PDF", - description = "This endpoint fetches content from a URL and converts it to a PDF format." + description = "This endpoint fetches content from a URL and converts it to a PDF format. Input:N/A Output:PDF Type:SISO" ) public ResponseEntity urlToPdf(@ModelAttribute UrlToPdfRequest request) throws IOException, InterruptedException { String URL = request.getUrlInput(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractController.java b/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractController.java index f6fb69c9..6398e8b9 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/converters/ExtractController.java @@ -28,12 +28,12 @@ import stirling.software.SPDF.model.api.extract.PDFFilePage; @RestController @RequestMapping("/api/v1/convert") -@Tag(name = "General", description = "General APIs") +@Tag(name = "Convert", description = "Convert APIs") public class ExtractController { private static final Logger logger = LoggerFactory.getLogger(CropController.class); - @PostMapping(value = "/pdf-to-csv", consumes = "multipart/form-data") + @PostMapping(value = "/pdf/csv", consumes = "multipart/form-data") @Operation(summary = "Extracts a PDF document to csv", description = "This operation takes an input PDF file and returns CSV file of whole page. Input:PDF Output:CSV Type:SISO") public ResponseEntity PdfToCsv(@ModelAttribute PDFFilePage form) throws Exception { diff --git a/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java b/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java index b4b9b951..e62fc35f 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/misc/AutoSplitPdfController.java @@ -43,7 +43,7 @@ public class AutoSplitPdfController { private static final String QR_CONTENT = "https://github.com/Frooodle/Stirling-PDF"; @PostMapping(value = "/auto-split-pdf", consumes = "multipart/form-data") - @Operation(summary = "Auto split PDF pages into separate documents", description = "This endpoint accepts a PDF file, scans each page for a specific QR code, and splits the document at the QR code boundaries. The output is a zip file containing each separate PDF document. Input:PDF Output:ZIP Type:SISO") + @Operation(summary = "Auto split PDF pages into separate documents", description = "This endpoint accepts a PDF file, scans each page for a specific QR code, and splits the document at the QR code boundaries. The output is a zip file containing each separate PDF document. Input:PDF Output:ZIP-PDF Type:SISO") public ResponseEntity autoSplitPdf(@ModelAttribute AutoSplitPdfRequest request) throws IOException { MultipartFile file = request.getFileInput(); boolean duplexMode = request.isDuplexMode(); diff --git a/src/main/java/stirling/software/SPDF/controller/api/misc/ShowJavascript.java b/src/main/java/stirling/software/SPDF/controller/api/misc/ShowJavascript.java index 27431346..cef32ac7 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/misc/ShowJavascript.java +++ b/src/main/java/stirling/software/SPDF/controller/api/misc/ShowJavascript.java @@ -15,6 +15,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import stirling.software.SPDF.model.api.PDFFile; import stirling.software.SPDF.utils.WebResponseUtils; @@ -25,6 +26,7 @@ public class ShowJavascript { private static final Logger logger = LoggerFactory.getLogger(ShowJavascript.class); @PostMapping(consumes = "multipart/form-data", value = "/show-javascript") + @Operation(summary = "Grabs all JS from a PDF and returns a single JS file with all code", description = "desc. Input:PDF Output:JS Type:SISO") public ResponseEntity extractHeader(@ModelAttribute PDFFile request) throws Exception { MultipartFile inputFile = request.getFileInput(); String script = ""; diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java new file mode 100644 index 00000000..43448f89 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/ApiDocService.java @@ -0,0 +1,119 @@ +package stirling.software.SPDF.controller.api.pipeline; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.servlet.ServletContext; +import stirling.software.SPDF.model.ApiEndpoint; +import stirling.software.SPDF.model.Role; +@Service +public class ApiDocService { + + private final Map apiDocumentation = new HashMap<>(); + + @Autowired + private ServletContext servletContext; + + private String getApiDocsUrl() { + String contextPath = servletContext.getContextPath(); + String port = System.getProperty("local.server.port"); + if(port == null || port.length() == 0) { + port="8080"; + } + + return "http://localhost:"+ port + contextPath + "/v1/api-docs"; + } + + + @Autowired(required=false) + private UserServiceInterface userService; + + private String getApiKeyForUser() { + if(userService == null) + return ""; + return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); + } + + JsonNode apiDocsJsonRootNode; + + + //@EventListener(ApplicationReadyEvent.class) + private synchronized void loadApiDocumentation() { + try { + HttpHeaders headers = new HttpHeaders(); + String apiKey = getApiKeyForUser(); + if (!apiKey.isEmpty()) { + headers.set("X-API-KEY", apiKey); + } + HttpEntity entity = new HttpEntity<>(headers); + + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity response = restTemplate.exchange(getApiDocsUrl(), HttpMethod.GET, entity, String.class); + String apiDocsJson = response.getBody(); + + ObjectMapper mapper = new ObjectMapper(); + apiDocsJsonRootNode = mapper.readTree(apiDocsJson); + + JsonNode paths = apiDocsJsonRootNode.path("paths"); + paths.fields().forEachRemaining(entry -> { + String path = entry.getKey(); + JsonNode pathNode = entry.getValue(); + if (pathNode.has("post")) { + JsonNode postNode = pathNode.get("post"); + ApiEndpoint endpoint = new ApiEndpoint(path, postNode); + apiDocumentation.put(path, endpoint); + } + }); + } catch (Exception e) { + // Handle exceptions + e.printStackTrace(); + } + } + + public boolean isValidOperation(String operationName, Map parameters) { + if(apiDocumentation.size() == 0) { + loadApiDocumentation(); + } + if (!apiDocumentation.containsKey(operationName)) { + return false; + } + ApiEndpoint endpoint = apiDocumentation.get(operationName); + return endpoint.areParametersValid(parameters); + } + + public boolean isMultiInput(String operationName) { + if(apiDocsJsonRootNode == null || apiDocumentation.size() == 0) { + loadApiDocumentation(); + } + if (!apiDocumentation.containsKey(operationName)) { + return false; + } + + ApiEndpoint endpoint = apiDocumentation.get(operationName); + String description = endpoint.getDescription(); + + Pattern pattern = Pattern.compile("Type:(\\w+)"); + Matcher matcher = pattern.matcher(description); + if (matcher.find()) { + String type = matcher.group(1); + return type.startsWith("MI"); + } + + return false; + } +} + +// Model class for API Endpoint + diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java index c12fe724..a6f78a6f 100644 --- a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineController.java @@ -1,56 +1,31 @@ package stirling.software.SPDF.controller.api.pipeline; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.stream.Stream; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.tags.Tag; import stirling.software.SPDF.model.ApplicationProperties; import stirling.software.SPDF.model.PipelineConfig; -import stirling.software.SPDF.model.PipelineOperation; import stirling.software.SPDF.model.api.HandleDataRequest; import stirling.software.SPDF.utils.WebResponseUtils; @@ -60,373 +35,39 @@ import stirling.software.SPDF.utils.WebResponseUtils; public class PipelineController { private static final Logger logger = LoggerFactory.getLogger(PipelineController.class); - @Autowired - private ObjectMapper objectMapper; - final String jsonFileName = "pipelineConfig.json"; final String watchedFoldersDir = "./pipeline/watchedFolders/"; final String finishedFoldersDir = "./pipeline/finishedFolders/"; + @Autowired + PipelineProcessor processor; - @Scheduled(fixedRate = 25000) - public void scanFolders() { - logger.info("Scanning folders..."); - Path watchedFolderPath = Paths.get(watchedFoldersDir); - if (!Files.exists(watchedFolderPath)) { - try { - Files.createDirectories(watchedFolderPath); - logger.info("Created directory: {}", watchedFolderPath); - } catch (IOException e) { - logger.error("Error creating directory: {}", watchedFolderPath, e); - return; - } - } - try (Stream paths = Files.walk(watchedFolderPath)) { - paths.filter(Files::isDirectory).forEach(t -> { - try { - if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) { - handleDirectory(t); - } - } catch (Exception e) { - logger.error("Error handling directory: {}", t, e); - } - }); - } catch (Exception e) { - logger.error("Error walking through directory: {}", watchedFolderPath, e); - } - } @Autowired ApplicationProperties applicationProperties; - - private void handleDirectory(Path dir) throws Exception { - logger.info("Handling directory: {}", dir); - Path jsonFile = dir.resolve(jsonFileName); - Path processingDir = dir.resolve("processing"); // Directory to move files during processing - if (!Files.exists(processingDir)) { - Files.createDirectory(processingDir); - logger.info("Created processing directory: {}", processingDir); - } - - if (Files.exists(jsonFile)) { - // Read JSON file - String jsonString; - try { - jsonString = new String(Files.readAllBytes(jsonFile)); - logger.info("Read JSON file: {}", jsonFile); - } catch (IOException e) { - logger.error("Error reading JSON file: {}", jsonFile, e); - return; - } - - // Decode JSON to PipelineConfig - PipelineConfig config; - try { - config = objectMapper.readValue(jsonString, PipelineConfig.class); - // Assuming your PipelineConfig class has getters for all necessary fields, you - // can perform checks here - if (config.getOperations() == null || config.getOutputDir() == null || config.getName() == null) { - throw new IOException("Invalid JSON format"); - } - } catch (IOException e) { - logger.error("Error parsing PipelineConfig: {}", jsonString, e); - return; - } - - // For each operation in the pipeline - for (PipelineOperation operation : config.getOperations()) { - // Collect all files based on fileInput - File[] files; - String fileInput = (String) operation.getParameters().get("fileInput"); - if ("automated".equals(fileInput)) { - // If fileInput is "automated", process all files in the directory - try (Stream paths = Files.list(dir)) { - files = paths - .filter(path -> !Files.isDirectory(path)) // exclude directories - .filter(path -> !path.equals(jsonFile)) // exclude jsonFile - .map(Path::toFile) - .toArray(File[]::new); - - } catch (IOException e) { - e.printStackTrace(); - return; - } - } else { - // If fileInput contains a path, process only this file - files = new File[] { new File(fileInput) }; - } - - // Prepare the files for processing - List filesToProcess = new ArrayList<>(); - for (File file : files) { - logger.info(file.getName()); - logger.info("{} to {}",file.toPath(), processingDir.resolve(file.getName())); - Files.move(file.toPath(), processingDir.resolve(file.getName())); - filesToProcess.add(processingDir.resolve(file.getName()).toFile()); - } - - // Process the files - try { - List resources = handleFiles(filesToProcess.toArray(new File[0]), jsonString); - - if(resources == null) { - return; - } - // Move resultant files and rename them as per config in JSON file - for (Resource resource : resources) { - String resourceName = resource.getFilename(); - String baseName = resourceName.substring(0, resourceName.lastIndexOf(".")); - String extension = resourceName.substring(resourceName.lastIndexOf(".")+1); - - String outputFileName = config.getOutputPattern().replace("{filename}", baseName); - - outputFileName = outputFileName.replace("{pipelineName}", config.getName()); - DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd"); - outputFileName = outputFileName.replace("{date}", LocalDate.now().format(dateFormatter)); - DateTimeFormatter timeFormatter = DateTimeFormatter.ofPattern("HHmmss"); - outputFileName = outputFileName.replace("{time}", LocalTime.now().format(timeFormatter)); - - outputFileName += "." + extension; - // {filename} {folder} {date} {tmime} {pipeline} - String outputDir = config.getOutputDir(); - - String outputFolder = applicationProperties.getAutoPipeline().getOutputFolder(); - - if (outputFolder == null || outputFolder.isEmpty()) { - // If the environment variable is not set, use the default value - outputFolder = finishedFoldersDir; - } - logger.info("outputDir 0={}", outputDir); - // Replace the placeholders in the outputDir string - outputDir = outputDir.replace("{outputFolder}", outputFolder); - outputDir = outputDir.replace("{folderName}", dir.toString()); - logger.info("outputDir 1={}", outputDir); - outputDir = outputDir.replace("\\watchedFolders", ""); - outputDir = outputDir.replace("//watchedFolders", ""); - outputDir = outputDir.replace("\\\\watchedFolders", ""); - outputDir = outputDir.replace("/watchedFolders", ""); - - Path outputPath; - logger.info("outputDir 2={}", outputDir); - if (Paths.get(outputDir).isAbsolute()) { - // If it's an absolute path, use it directly - outputPath = Paths.get(outputDir); - } else { - // If it's a relative path, make it relative to the current working directory - outputPath = Paths.get(".", outputDir); - } - - logger.info("outputPath={}", outputPath); - - if (!Files.exists(outputPath)) { - try { - Files.createDirectories(outputPath); - logger.info("Created directory: {}", outputPath); - } catch (IOException e) { - logger.error("Error creating directory: {}", outputPath, e); - return; - } - } - logger.info("outputPath {}", outputPath); - logger.info("outputPath.resolve(outputFileName).toString() {}", outputPath.resolve(outputFileName).toString()); - File newFile = new File(outputPath.resolve(outputFileName).toString()); - OutputStream os = new FileOutputStream(newFile); - os.write(((ByteArrayResource)resource).getByteArray()); - os.close(); - logger.info("made {}", outputPath.resolve(outputFileName)); - } - - // If successful, delete the original files - for (File file : filesToProcess) { - Files.deleteIfExists(processingDir.resolve(file.getName())); - } - } catch (Exception e) { - // If an error occurs, move the original files back - for (File file : filesToProcess) { - Files.move(processingDir.resolve(file.getName()), file.toPath()); - } - throw e; - } - } - } - } - - List processFiles(List outputFiles, String jsonString) throws Exception { - - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - logger.info("Running pipelineNode: {}", pipelineNode); - ByteArrayOutputStream logStream = new ByteArrayOutputStream(); - PrintStream logPrintStream = new PrintStream(logStream); - - boolean hasErrors = false; - - for (JsonNode operationNode : pipelineNode) { - String operation = operationNode.get("operation").asText(); - logger.info("Running operation: {}", operation); - JsonNode parametersNode = operationNode.get("parameters"); - String inputFileExtension = ""; - if (operationNode.has("inputFileType")) { - inputFileExtension = operationNode.get("inputFileType").asText(); - } else { - inputFileExtension = ".pdf"; - } - - List newOutputFiles = new ArrayList<>(); - boolean hasInputFileType = false; - - for (Resource file : outputFiles) { - if (file.getFilename().endsWith(inputFileExtension)) { - hasInputFileType = true; - MultiValueMap body = new LinkedMultiValueMap<>(); - body.add("fileInput", file); - - Iterator> parameters = parametersNode.fields(); - while (parameters.hasNext()) { - Map.Entry parameter = parameters.next(); - body.add(parameter.getKey(), parameter.getValue().asText()); - } - - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); - - HttpEntity> entity = new HttpEntity<>(body, headers); - - RestTemplate restTemplate = new RestTemplate(); - String url = "http://localhost:8080/" + operation; - - ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); - - // If the operation is filter and the response body is null or empty, skip this file - if (operation.startsWith("filter-") && (response.getBody() == null || response.getBody().length == 0)) { - logger.info("Skipping file due to failing {}", operation); - continue; - } - - if (!response.getStatusCode().equals(HttpStatus.OK)) { - logPrintStream.println("Error: " + response.getBody()); - hasErrors = true; - continue; - } - - - // Define filename - String filename; - if ("auto-rename".equals(operation)) { - // If the operation is "auto-rename", generate a new filename. - // This is a simple example of generating a filename using current timestamp. - // Modify as per your needs. - filename = "file_" + System.currentTimeMillis(); - } else { - // Otherwise, keep the original filename. - filename = file.getFilename(); - } - - // Check if the response body is a zip file - if (isZip(response.getBody())) { - // Unzip the file and add all the files to the new output files - newOutputFiles.addAll(unzip(response.getBody())); - } else { - Resource outputResource = new ByteArrayResource(response.getBody()) { - @Override - public String getFilename() { - return filename; - } - }; - newOutputFiles.add(outputResource); - } - } - - if (!hasInputFileType) { - logPrintStream.println( - "No files with extension " + inputFileExtension + " found for operation " + operation); - hasErrors = true; - } - - outputFiles = newOutputFiles; - } - logPrintStream.close(); - - } - if (hasErrors) { - logger.error("Errors occurred during processing. Log: {}", logStream.toString()); - } - return outputFiles; - } - - List handleFiles(File[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { - logger.info("No files"); - return null; - } - - logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - - boolean hasErrors = false; - List outputFiles = new ArrayList<>(); - - for (File file : files) { - Path path = Paths.get(file.getAbsolutePath()); - System.out.println("Reading file: " + path); // debug statement - - if (Files.exists(path)) { - Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { - @Override - public String getFilename() { - return file.getName(); - } - }; - outputFiles.add(fileResource); - } else { - System.out.println("File not found: " + path); // debug statement - } - } - logger.info("Files successfully loaded. Starting processing..."); - return processFiles(outputFiles, jsonString); - } - - List handleFiles(MultipartFile[] files, String jsonString) throws Exception { - if(files == null || files.length == 0) { - logger.info("No files"); - return null; - } - logger.info("Handling files: {} files, with JSON string of length: {}", files.length, jsonString.length()); - ObjectMapper mapper = new ObjectMapper(); - JsonNode jsonNode = mapper.readTree(jsonString); - - JsonNode pipelineNode = jsonNode.get("pipeline"); - - boolean hasErrors = false; - List outputFiles = new ArrayList<>(); - - for (MultipartFile file : files) { - Resource fileResource = new ByteArrayResource(file.getBytes()) { - @Override - public String getFilename() { - return file.getOriginalFilename(); - } - }; - outputFiles.add(fileResource); - } - logger.info("Files successfully loaded. Starting processing..."); - return processFiles(outputFiles, jsonString); - } + @Autowired + private ObjectMapper objectMapper; + @PostMapping("/handleData") - public ResponseEntity handleData(@ModelAttribute HandleDataRequest request) { - MultipartFile[] files = request.getFileInputs(); - String jsonString = request.getJsonString(); + public ResponseEntity handleData(@ModelAttribute HandleDataRequest request) throws JsonMappingException, JsonProcessingException { + if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { + return new ResponseEntity<>(HttpStatus.BAD_REQUEST); + } + + MultipartFile[] files = request.getFileInput(); + String jsonString = request.getJson(); + if (files == null) { + return null; + } + PipelineConfig config = objectMapper.readValue(jsonString, PipelineConfig.class); logger.info("Received POST request to /handleData with {} files", files.length); try { - List outputFiles = handleFiles(files, jsonString); - + List inputFiles = processor.generateInputFiles(files); + if(inputFiles == null || inputFiles.size() == 0) { + return null; + } + List outputFiles = processor.runPipelineAgainstFiles(inputFiles, config); if (outputFiles != null && outputFiles.size() == 1) { // If there is only one file, return it directly Resource singleFile = outputFiles.get(0); @@ -473,52 +114,4 @@ public class PipelineController { } } - private boolean isZip(byte[] data) { - if (data == null || data.length < 4) { - return false; - } - - // Check the first four bytes of the data against the standard zip magic number - return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04; - } - - private List unzip(byte[] data) throws IOException { - logger.info("Unzipping data of length: {}", data.length); - List unzippedFiles = new ArrayList<>(); - - try (ByteArrayInputStream bais = new ByteArrayInputStream(data); - ZipInputStream zis = new ZipInputStream(bais)) { - - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int count; - - while ((count = zis.read(buffer)) != -1) { - baos.write(buffer, 0, count); - } - - final String filename = entry.getName(); - Resource fileResource = new ByteArrayResource(baos.toByteArray()) { - @Override - public String getFilename() { - return filename; - } - }; - - // If the unzipped file is a zip file, unzip it - if (isZip(baos.toByteArray())) { - logger.info("File {} is a zip file. Unzipping...", filename); - unzippedFiles.addAll(unzip(baos.toByteArray())); - } else { - unzippedFiles.add(fileResource); - } - } - } - - logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size()); - return unzippedFiles; - } - } diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java new file mode 100644 index 00000000..dc45d4cb --- /dev/null +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineDirectoryProcessor.java @@ -0,0 +1,258 @@ +package stirling.software.SPDF.controller.api.pipeline; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import stirling.software.SPDF.model.ApplicationProperties; +import stirling.software.SPDF.model.PipelineConfig; +import stirling.software.SPDF.model.PipelineOperation; + +@Service +public class PipelineDirectoryProcessor { + + private static final Logger logger = LoggerFactory.getLogger(PipelineDirectoryProcessor.class); + @Autowired + private ObjectMapper objectMapper; + @Autowired + private ApiDocService apiDocService; + @Autowired + private ApplicationProperties applicationProperties; + + final String watchedFoldersDir = "./pipeline/watchedFolders/"; + final String finishedFoldersDir = "./pipeline/finishedFolders/"; + + @Autowired + PipelineProcessor processor; + + @Scheduled(fixedRate = 60000) + public void scanFolders() { + if (!Boolean.TRUE.equals(applicationProperties.getSystem().getEnableAlphaFunctionality())) { + return; + } + Path watchedFolderPath = Paths.get(watchedFoldersDir); + if (!Files.exists(watchedFolderPath)) { + try { + Files.createDirectories(watchedFolderPath); + logger.info("Created directory: {}", watchedFolderPath); + } catch (IOException e) { + logger.error("Error creating directory: {}", watchedFolderPath, e); + return; + } + } + try (Stream paths = Files.walk(watchedFolderPath)) { + paths.filter(Files::isDirectory).forEach(t -> { + try { + if (!t.equals(watchedFolderPath) && !t.endsWith("processing")) { + handleDirectory(t); + } + } catch (Exception e) { + logger.error("Error handling directory: {}", t, e); + } + }); + } catch (Exception e) { + logger.error("Error walking through directory: {}", watchedFolderPath, e); + } + } + + public void handleDirectory(Path dir) throws IOException { + logger.info("Handling directory: {}", dir); + Path processingDir = createProcessingDirectory(dir); + + Optional jsonFileOptional = findJsonFile(dir); + if (!jsonFileOptional.isPresent()) { + logger.warn("No .JSON settings file found. No processing will happen for dir {}.", dir); + return; + } + + Path jsonFile = jsonFileOptional.get(); + PipelineConfig config = readAndParseJson(jsonFile); + processPipelineOperations(dir, processingDir, jsonFile, config); + } + + private Path createProcessingDirectory(Path dir) throws IOException { + Path processingDir = dir.resolve("processing"); + if (!Files.exists(processingDir)) { + Files.createDirectory(processingDir); + logger.info("Created processing directory: {}", processingDir); + } + return processingDir; + } + + private Optional findJsonFile(Path dir) throws IOException { + try (Stream paths = Files.list(dir)) { + return paths.filter(file -> file.toString().endsWith(".json")).findFirst(); + } + } + + private PipelineConfig readAndParseJson(Path jsonFile) throws IOException { + String jsonString = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); + logger.debug("Reading JSON file: {}", jsonFile); + return objectMapper.readValue(jsonString, PipelineConfig.class); + } + + private void processPipelineOperations(Path dir, Path processingDir, Path jsonFile, PipelineConfig config) throws IOException { + for (PipelineOperation operation : config.getOperations()) { + validateOperation(operation); + File[] files = collectFilesForProcessing(dir, jsonFile, operation); + if(files == null || files.length == 0) { + logger.debug("No files detected for {} ", dir); + return; + } + List filesToProcess = prepareFilesForProcessing(files, processingDir); + runPipelineAgainstFiles(filesToProcess, config, dir, processingDir); + } + } + + private void validateOperation(PipelineOperation operation) throws IOException { + if (!apiDocService.isValidOperation(operation.getOperation(), operation.getParameters())) { + throw new IOException("Invalid operation: " + operation.getOperation()); + } + } + + private File[] collectFilesForProcessing(Path dir, Path jsonFile, PipelineOperation operation) throws IOException { + try (Stream paths = Files.list(dir)) { + if ("automated".equals(operation.getParameters().get("fileInput"))) { + return paths.filter(path -> !Files.isDirectory(path) && !path.equals(jsonFile)) + .map(Path::toFile) + .toArray(File[]::new); + } else { + String fileInput = (String) operation.getParameters().get("fileInput"); + return new File[]{new File(fileInput)}; + } + } + } + + private List prepareFilesForProcessing(File[] files, Path processingDir) throws IOException { + List filesToProcess = new ArrayList<>(); + for (File file : files) { + Path targetPath = resolveUniqueFilePath(processingDir, file.getName()); + Files.move(file.toPath(), targetPath); + filesToProcess.add(targetPath.toFile()); + } + return filesToProcess; + } + + private Path resolveUniqueFilePath(Path directory, String originalFileName) { + Path filePath = directory.resolve(originalFileName); + int counter = 1; + + while (Files.exists(filePath)) { + String newName = appendSuffixToFileName(originalFileName, "(" + counter + ")"); + filePath = directory.resolve(newName); + counter++; + } + + return filePath; + } + + private String appendSuffixToFileName(String originalFileName, String suffix) { + int dotIndex = originalFileName.lastIndexOf('.'); + if (dotIndex == -1) { + return originalFileName + suffix; + } else { + return originalFileName.substring(0, dotIndex) + suffix + originalFileName.substring(dotIndex); + } + } + + private void runPipelineAgainstFiles(List filesToProcess, PipelineConfig config, Path dir, Path processingDir) throws IOException { + try { + List inputFiles = processor.generateInputFiles(filesToProcess.toArray(new File[0])); + if(inputFiles == null || inputFiles.size() == 0) { + return; + } + List outputFiles = processor.runPipelineAgainstFiles(inputFiles, config); + if (outputFiles == null) return; + moveAndRenameFiles(outputFiles, config, dir); + deleteOriginalFiles(filesToProcess, processingDir); + } catch (Exception e) { + logger.error("error during processing", e); + moveFilesBack(filesToProcess, processingDir); + } + } + + private void moveAndRenameFiles(List resources, PipelineConfig config, Path dir) throws IOException { + for (Resource resource : resources) { + String outputFileName = createOutputFileName(resource, config); + Path outputPath = determineOutputPath(config, dir); + + if (!Files.exists(outputPath)) { + Files.createDirectories(outputPath); + logger.info("Created directory: {}", outputPath); + } + + Path outputFile = outputPath.resolve(outputFileName); + try (OutputStream os = new FileOutputStream(outputFile.toFile())) { + os.write(((ByteArrayResource) resource).getByteArray()); + } + + logger.info("File moved and renamed to {}", outputFile); + } + } + + private String createOutputFileName(Resource resource, PipelineConfig config) { + String resourceName = resource.getFilename(); + String baseName = resourceName.substring(0, resourceName.lastIndexOf('.')); + String extension = resourceName.substring(resourceName.lastIndexOf('.') + 1); + + String outputFileName = config.getOutputPattern() + .replace("{filename}", baseName) + .replace("{pipelineName}", config.getName()) + .replace("{date}", LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"))) + .replace("{time}", LocalTime.now().format(DateTimeFormatter.ofPattern("HHmmss"))) + + "." + extension; + + return outputFileName; + } + + private Path determineOutputPath(PipelineConfig config, Path dir) { + String outputDir = config.getOutputDir() + .replace("{outputFolder}", finishedFoldersDir) + .replace("{folderName}", dir.toString()) + .replaceAll("\\\\?watchedFolders", ""); + + return Paths.get(outputDir).isAbsolute() ? Paths.get(outputDir) : Paths.get(".", outputDir); + } + + private void deleteOriginalFiles(List filesToProcess, Path processingDir) throws IOException { + for (File file : filesToProcess) { + Files.deleteIfExists(processingDir.resolve(file.getName())); + logger.info("Deleted original file: {}", file.getName()); + } + } + + private void moveFilesBack(List filesToProcess, Path processingDir) { + for (File file : filesToProcess) { + try { + Files.move(processingDir.resolve(file.getName()), file.toPath()); + logger.info("Moved file back to original location: {} , {}",file.toPath(), file.getName()); + } catch (IOException e) { + logger.error("Error moving file back to original location: {}", file.getName(), e); + } + } + } + + +} diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java new file mode 100644 index 00000000..1ac7eef9 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/PipelineProcessor.java @@ -0,0 +1,328 @@ +package stirling.software.SPDF.controller.api.pipeline; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import jakarta.servlet.ServletContext; +import stirling.software.SPDF.model.PipelineConfig; +import stirling.software.SPDF.model.PipelineOperation; +import stirling.software.SPDF.model.Role; + +@Service +public class PipelineProcessor { + + private static final Logger logger = LoggerFactory.getLogger(PipelineProcessor.class); + + + @Autowired + private ApiDocService apiDocService; + + @Autowired + private UserServiceInterface userService; + + @Autowired + private ServletContext servletContext; + + + + + private String getApiKeyForUser() { + if (userService == null) + return ""; + return userService.getApiKeyForUser(Role.INTERNAL_API_USER.getRoleId()); + } + + + private String getBaseUrl() { + String contextPath = servletContext.getContextPath(); + return "http://localhost:8080" + contextPath + "/"; + } + + + + List runPipelineAgainstFiles(List outputFiles, PipelineConfig config) throws Exception { + + ByteArrayOutputStream logStream = new ByteArrayOutputStream(); + PrintStream logPrintStream = new PrintStream(logStream); + + boolean hasErrors = false; + + for (PipelineOperation pipelineOperation : config.getOperations()) { + String operation = pipelineOperation.getOperation(); + boolean isMultiInputOperation = apiDocService.isMultiInput(operation); + + logger.info("Running operation: {} isMultiInputOperation {}", operation, isMultiInputOperation); + Map parameters = pipelineOperation.getParameters(); + String inputFileExtension = ""; + + //TODO + //if (operationNode.has("inputFileType")) { + // inputFileExtension = operationNode.get("inputFileType").asText(); + //} else { + inputFileExtension = ".pdf"; + //} + final String finalInputFileExtension = inputFileExtension; + + String url = getBaseUrl() + operation; + + List newOutputFiles = new ArrayList<>(); + if (!isMultiInputOperation) { + for (Resource file : outputFiles) { + boolean hasInputFileType = false; + if (file.getFilename().endsWith(inputFileExtension)) { + hasInputFileType = true; + MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("fileInput", file); + + + for(Entry entry : parameters.entrySet()) { + body.add(entry.getKey(), entry.getValue()); + } + + ResponseEntity response = sendWebRequest(url, body); + + // If the operation is filter and the response body is null or empty, skip this + // file + if (operation.startsWith("filter-") + && (response.getBody() == null || response.getBody().length == 0)) { + logger.info("Skipping file due to failing {}", operation); + continue; + } + + if (!response.getStatusCode().equals(HttpStatus.OK)) { + logPrintStream.println("Error: " + response.getBody()); + hasErrors = true; + continue; + } + processOutputFiles(operation, file.getFilename(), response, newOutputFiles); + + } + + if (!hasInputFileType) { + logPrintStream.println( + "No files with extension " + inputFileExtension + " found for operation " + operation); + hasErrors = true; + } + + outputFiles = newOutputFiles; + } + + } else { + // Filter and collect all files that match the inputFileExtension + List matchingFiles = outputFiles.stream() + .filter(file -> file.getFilename().endsWith(finalInputFileExtension)) + .collect(Collectors.toList()); + + // Check if there are matching files + if (!matchingFiles.isEmpty()) { + // Create a new MultiValueMap for the request body + MultiValueMap body = new LinkedMultiValueMap<>(); + + // Add all matching files to the body + for (Resource file : matchingFiles) { + body.add("fileInput", file); + } + + for(Entry entry : parameters.entrySet()) { + body.add(entry.getKey(), entry.getValue()); + } + + ResponseEntity response = sendWebRequest(url, body); + + // Handle the response + if (response.getStatusCode().equals(HttpStatus.OK)) { + processOutputFiles(operation, matchingFiles.get(0).getFilename(), response, newOutputFiles); + } else { + // Log error if the response status is not OK + logPrintStream.println("Error in multi-input operation: " + response.getBody()); + hasErrors = true; + } + } else { + logPrintStream.println("No files with extension " + inputFileExtension + " found for multi-input operation " + operation); + hasErrors = true; + } + } + logPrintStream.close(); + + } + if (hasErrors) { + logger.error("Errors occurred during processing. Log: {}", logStream.toString()); + } + return outputFiles; + } + + private ResponseEntity sendWebRequest(String url, MultiValueMap body ){ + RestTemplate restTemplate = new RestTemplate(); + + // Set up headers, including API key + HttpHeaders headers = new HttpHeaders(); + String apiKey = getApiKeyForUser(); + headers.add("X-API-Key", apiKey); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + + // Create HttpEntity with the body and headers + HttpEntity> entity = new HttpEntity<>(body, headers); + + // Make the request to the REST endpoint + return restTemplate.exchange(url, HttpMethod.POST, entity, byte[].class); + } + + private List processOutputFiles(String operation, String fileName, ResponseEntity response, List newOutputFiles) throws IOException{ + // Define filename + String newFilename; + if ("auto-rename".equals(operation)) { + // If the operation is "auto-rename", generate a new filename. + // This is a simple example of generating a filename using current timestamp. + // Modify as per your needs. + newFilename = "file_" + System.currentTimeMillis(); + } else { + // Otherwise, keep the original filename. + newFilename = fileName; + } + + // Check if the response body is a zip file + if (isZip(response.getBody())) { + // Unzip the file and add all the files to the new output files + newOutputFiles.addAll(unzip(response.getBody())); + } else { + Resource outputResource = new ByteArrayResource(response.getBody()) { + @Override + public String getFilename() { + return newFilename; + } + }; + newOutputFiles.add(outputResource); + } + + return newOutputFiles; + + } + List generateInputFiles(File[] files) throws Exception { + if (files == null || files.length == 0) { + logger.info("No files"); + return null; + } + + + List outputFiles = new ArrayList<>(); + + for (File file : files) { + Path path = Paths.get(file.getAbsolutePath()); + logger.info("Reading file: " + path); // debug statement + + if (Files.exists(path)) { + Resource fileResource = new ByteArrayResource(Files.readAllBytes(path)) { + @Override + public String getFilename() { + return file.getName(); + } + }; + outputFiles.add(fileResource); + } else { + logger.info("File not found: " + path); + } + } + logger.info("Files successfully loaded. Starting processing..."); + return outputFiles; + } + + List generateInputFiles(MultipartFile[] files) throws Exception { + if (files == null || files.length == 0) { + logger.info("No files"); + return null; + } + + List outputFiles = new ArrayList<>(); + + for (MultipartFile file : files) { + Resource fileResource = new ByteArrayResource(file.getBytes()) { + @Override + public String getFilename() { + return file.getOriginalFilename(); + } + }; + outputFiles.add(fileResource); + } + logger.info("Files successfully loaded. Starting processing..."); + return outputFiles; + } + + private boolean isZip(byte[] data) { + if (data == null || data.length < 4) { + return false; + } + + // Check the first four bytes of the data against the standard zip magic number + return data[0] == 0x50 && data[1] == 0x4B && data[2] == 0x03 && data[3] == 0x04; + } + + private List unzip(byte[] data) throws IOException { + logger.info("Unzipping data of length: {}", data.length); + List unzippedFiles = new ArrayList<>(); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(data); + ZipInputStream zis = new ZipInputStream(bais)) { + + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int count; + + while ((count = zis.read(buffer)) != -1) { + baos.write(buffer, 0, count); + } + + final String filename = entry.getName(); + Resource fileResource = new ByteArrayResource(baos.toByteArray()) { + @Override + public String getFilename() { + return filename; + } + }; + + // If the unzipped file is a zip file, unzip it + if (isZip(baos.toByteArray())) { + logger.info("File {} is a zip file. Unzipping...", filename); + unzippedFiles.addAll(unzip(baos.toByteArray())); + } else { + unzippedFiles.add(fileResource); + } + } + } + + logger.info("Unzipping completed. {} files were unzipped.", unzippedFiles.size()); + return unzippedFiles; + } + +} diff --git a/src/main/java/stirling/software/SPDF/controller/api/pipeline/UserServiceInterface.java b/src/main/java/stirling/software/SPDF/controller/api/pipeline/UserServiceInterface.java new file mode 100644 index 00000000..f1203be8 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/controller/api/pipeline/UserServiceInterface.java @@ -0,0 +1,4 @@ +package stirling.software.SPDF.controller.api.pipeline; +public interface UserServiceInterface { + String getApiKeyForUser(String username); +} diff --git a/src/main/java/stirling/software/SPDF/controller/web/AccountWebController.java b/src/main/java/stirling/software/SPDF/controller/web/AccountWebController.java index cba7ebb5..2c51e7d8 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/AccountWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/AccountWebController.java @@ -1,4 +1,5 @@ package stirling.software.SPDF.controller.web; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -15,6 +16,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletRequest; +import stirling.software.SPDF.model.Authority; +import stirling.software.SPDF.model.Role; import stirling.software.SPDF.model.User; import stirling.software.SPDF.repository.UserRepository; @Controller @@ -46,14 +49,28 @@ public class AccountWebController { @PreAuthorize("hasRole('ROLE_ADMIN')") @GetMapping("/addUsers") public String showAddUserForm(Model model, Authentication authentication) { - List allUsers = userRepository.findAll(); + List allUsers = userRepository.findAll(); + Iterator iterator = allUsers.iterator(); + + while(iterator.hasNext()) { + User user = iterator.next(); + if(user != null) { + for (Authority authority : user.getAuthorities()) { + if (authority.getAuthority().equals(Role.INTERNAL_API_USER.getRoleId())) { + iterator.remove(); + break; // Break out of the inner loop once the user is removed + } + } + } + } + model.addAttribute("users", allUsers); model.addAttribute("currentUsername", authentication.getName()); return "addUsers"; } - + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @GetMapping("/account") public String account(HttpServletRequest request, Model model, Authentication authentication) { if (authentication == null || !authentication.isAuthenticated()) { @@ -100,7 +117,7 @@ public class AccountWebController { } - + @PreAuthorize("!hasAuthority('ROLE_DEMO_USER')") @GetMapping("/change-creds") public String changeCreds(HttpServletRequest request, Model model, Authentication authentication) { if (authentication == null || !authentication.isAuthenticated()) { diff --git a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java index 4b038e52..b670a129 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/GeneralWebController.java @@ -1,5 +1,6 @@ package stirling.software.SPDF.controller.web; +import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -41,31 +42,44 @@ public class GeneralWebController { model.addAttribute("currentPage", "pipeline"); List pipelineConfigs = new ArrayList<>(); - try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { - List jsonFiles = paths - .filter(Files::isRegularFile) - .filter(p -> p.toString().endsWith(".json")) - .collect(Collectors.toList()); - - for (Path jsonFile : jsonFiles) { - String content = Files.readString(jsonFile, StandardCharsets.UTF_8); - pipelineConfigs.add(content); - } - List> pipelineConfigsWithNames = new ArrayList<>(); - for (String config : pipelineConfigs) { - Map jsonContent = new ObjectMapper().readValue(config, new TypeReference>(){}); - - String name = (String) jsonContent.get("name"); - Map configWithName = new HashMap<>(); - configWithName.put("json", config); - configWithName.put("name", name); + List> pipelineConfigsWithNames = new ArrayList<>(); + + if(new File("./pipeline/defaultWebUIConfigs/").exists()) { + try (Stream paths = Files.walk(Paths.get("./pipeline/defaultWebUIConfigs/"))) { + List jsonFiles = paths + .filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".json")) + .collect(Collectors.toList()); + + for (Path jsonFile : jsonFiles) { + String content = Files.readString(jsonFile, StandardCharsets.UTF_8); + pipelineConfigs.add(content); + } + + for (String config : pipelineConfigs) { + Map jsonContent = new ObjectMapper().readValue(config, new TypeReference>(){}); + + String name = (String) jsonContent.get("name"); + Map configWithName = new HashMap<>(); + configWithName.put("json", config); + configWithName.put("name", name); + pipelineConfigsWithNames.add(configWithName); + } + + + + + } catch (IOException e) { + e.printStackTrace(); + } + } + if(pipelineConfigsWithNames.size() == 0) { + Map configWithName = new HashMap<>(); + configWithName.put("json", ""); + configWithName.put("name", "No preloaded configs found"); pipelineConfigsWithNames.add(configWithName); - } - model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); - - } catch (IOException e) { - e.printStackTrace(); } + model.addAttribute("pipelineConfigsWithNames", pipelineConfigsWithNames); model.addAttribute("pipelineConfigs", pipelineConfigs); diff --git a/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java b/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java index 9dc4f2b7..dbb254a7 100644 --- a/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java +++ b/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java @@ -27,8 +27,8 @@ import stirling.software.SPDF.config.StartupApplicationListener; import stirling.software.SPDF.model.ApplicationProperties; @RestController -@RequestMapping("/api/v1") -@Tag(name = "API", description = "Info APIs") +@RequestMapping("/api/v1/info") +@Tag(name = "Info", description = "Info APIs") public class MetricsController { diff --git a/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java b/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java new file mode 100644 index 00000000..1838b763 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/model/ApiEndpoint.java @@ -0,0 +1,42 @@ +package stirling.software.SPDF.model; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; + +public class ApiEndpoint { + private String name; + private Map parameters; + private String description; + + public ApiEndpoint(String name, JsonNode postNode) { + this.name = name; + this.parameters = new HashMap<>(); + postNode.path("parameters").forEach(paramNode -> { + String paramName = paramNode.path("name").asText(); + parameters.put(paramName, paramNode); + }); + this.description = postNode.path("description").asText(); + } + + public boolean areParametersValid(Map providedParams) { + for (String requiredParam : parameters.keySet()) { + if (!providedParams.containsKey(requiredParam)) { + return false; + } + } + return true; + } + + public String getDescription() { + return description; + } + + @Override + public String toString() { + return "ApiEndpoint [name=" + name + ", parameters=" + parameters + "]"; + } + + +} \ No newline at end of file diff --git a/src/main/java/stirling/software/SPDF/model/ApplicationProperties.java b/src/main/java/stirling/software/SPDF/model/ApplicationProperties.java index dc068779..36073c88 100644 --- a/src/main/java/stirling/software/SPDF/model/ApplicationProperties.java +++ b/src/main/java/stirling/software/SPDF/model/ApplicationProperties.java @@ -106,6 +106,25 @@ public class ApplicationProperties { private Boolean enableLogin; private Boolean csrfDisabled; private InitialLogin initialLogin; + private int loginAttemptCount; + private long loginResetTimeMinutes; + + + public int getLoginAttemptCount() { + return loginAttemptCount; + } + + public void setLoginAttemptCount(int loginAttemptCount) { + this.loginAttemptCount = loginAttemptCount; + } + + public long getLoginResetTimeMinutes() { + return loginResetTimeMinutes; + } + + public void setLoginResetTimeMinutes(long loginResetTimeMinutes) { + this.loginResetTimeMinutes = loginResetTimeMinutes; + } public InitialLogin getInitialLogin() { return initialLogin != null ? initialLogin : new InitialLogin(); @@ -175,6 +194,19 @@ public class ApplicationProperties { private String rootURIPath; private String customStaticFilePath; private Integer maxFileSize; + + private Boolean enableAlphaFunctionality; + + + + + public Boolean getEnableAlphaFunctionality() { + return enableAlphaFunctionality; + } + + public void setEnableAlphaFunctionality(Boolean enableAlphaFunctionality) { + this.enableAlphaFunctionality = enableAlphaFunctionality; + } public String getDefaultLocale() { return defaultLocale; @@ -218,12 +250,13 @@ public class ApplicationProperties { @Override public String toString() { - return "System [defaultLocale=" + defaultLocale + ", googlevisibility=" + googlevisibility + ", rootURIPath=" - + rootURIPath + ", customStaticFilePath=" + customStaticFilePath + ", maxFileSize=" + maxFileSize - + "]"; + return "System [defaultLocale=" + defaultLocale + ", googlevisibility=" + googlevisibility + + ", rootURIPath=" + rootURIPath + ", customStaticFilePath=" + customStaticFilePath + + ", maxFileSize=" + maxFileSize + ", enableAlphaFunctionality=" + enableAlphaFunctionality + "]"; } + } public static class Ui { diff --git a/src/main/java/stirling/software/SPDF/model/AttemptCounter.java b/src/main/java/stirling/software/SPDF/model/AttemptCounter.java new file mode 100644 index 00000000..fd668b05 --- /dev/null +++ b/src/main/java/stirling/software/SPDF/model/AttemptCounter.java @@ -0,0 +1,27 @@ +package stirling.software.SPDF.model; +public class AttemptCounter { + private int attemptCount; + private long lastAttemptTime; + + public AttemptCounter() { + this.attemptCount = 1; + this.lastAttemptTime = System.currentTimeMillis(); + } + + public void increment() { + this.attemptCount++; + this.lastAttemptTime = System.currentTimeMillis(); + } + + public int getAttemptCount() { + return attemptCount; + } + + public long getlastAttemptTime() { + return lastAttemptTime; + } + + public boolean shouldReset(long ATTEMPT_INCREMENT_TIME) { + return System.currentTimeMillis() - lastAttemptTime > ATTEMPT_INCREMENT_TIME; + } +} diff --git a/src/main/java/stirling/software/SPDF/model/Role.java b/src/main/java/stirling/software/SPDF/model/Role.java index 1b775de0..85315a16 100644 --- a/src/main/java/stirling/software/SPDF/model/Role.java +++ b/src/main/java/stirling/software/SPDF/model/Role.java @@ -14,8 +14,13 @@ public enum Role { EXTRA_LIMITED_API_USER("ROLE_EXTRA_LIMITED_API_USER", 20, 20), // 0 API calls per day and 20 web calls - WEB_ONLY_USER("ROLE_WEB_ONLY_USER", 0, 20); + WEB_ONLY_USER("ROLE_WEB_ONLY_USER", 0, 20), + + + INTERNAL_API_USER("STIRLING-PDF-BACKEND-API-USER", Integer.MAX_VALUE, Integer.MAX_VALUE), + DEMO_USER("ROLE_DEMO_USER", 100, 100); + private final String roleId; private final int apiCallsPerDay; private final int webCallsPerDay; diff --git a/src/main/java/stirling/software/SPDF/model/api/HandleDataRequest.java b/src/main/java/stirling/software/SPDF/model/api/HandleDataRequest.java index 1d7a8afe..d830ffeb 100644 --- a/src/main/java/stirling/software/SPDF/model/api/HandleDataRequest.java +++ b/src/main/java/stirling/software/SPDF/model/api/HandleDataRequest.java @@ -13,8 +13,8 @@ import lombok.NoArgsConstructor; public class HandleDataRequest { @Schema(description = "The input files") - private MultipartFile[] fileInputs; + private MultipartFile[] fileInput; @Schema(description = "JSON String") - private String jsonString; + private String json; } diff --git a/src/main/java/stirling/software/SPDF/model/api/PDFWithPageNums.java b/src/main/java/stirling/software/SPDF/model/api/PDFWithPageNums.java index d53d8d12..34542da8 100644 --- a/src/main/java/stirling/software/SPDF/model/api/PDFWithPageNums.java +++ b/src/main/java/stirling/software/SPDF/model/api/PDFWithPageNums.java @@ -5,6 +5,7 @@ import java.util.List; import org.apache.pdfbox.pdmodel.PDDocument; +import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; @@ -18,7 +19,7 @@ public class PDFWithPageNums extends PDFFile { @Schema(description = "The pages to select, Supports ranges (e.g., '1,3,5-9'), or 'all' or functions in the format 'an+b' where 'a' is the multiplier of the page number 'n', and 'b' is a constant (e.g., '2n+1', '3n', '6n-5')\"") private String pageNumbers; - + @Hidden public List getPageNumbersList(){ int pageCount = 0; try { @@ -30,6 +31,8 @@ public class PDFWithPageNums extends PDFFile { return GeneralUtils.parsePageString(pageNumbers, pageCount); } + + @Hidden public List getPageNumbersList(PDDocument doc){ int pageCount = 0; pageCount = doc.getNumberOfPages(); diff --git a/src/main/java/stirling/software/SPDF/utils/PdfUtils.java b/src/main/java/stirling/software/SPDF/utils/PdfUtils.java index 2be68d08..038b8302 100644 --- a/src/main/java/stirling/software/SPDF/utils/PdfUtils.java +++ b/src/main/java/stirling/software/SPDF/utils/PdfUtils.java @@ -4,22 +4,17 @@ import java.awt.Graphics; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import javax.imageio.ImageIO; import javax.imageio.IIOImage; +import javax.imageio.ImageIO; import javax.imageio.ImageReader; -import javax.imageio.ImageWriter; import javax.imageio.ImageWriteParam; +import javax.imageio.ImageWriter; import javax.imageio.stream.ImageOutputStream; import org.apache.pdfbox.pdmodel.PDDocument; diff --git a/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java new file mode 100644 index 00000000..0046ee9f --- /dev/null +++ b/src/main/java/stirling/software/SPDF/utils/RequestUriUtils.java @@ -0,0 +1,16 @@ +package stirling.software.SPDF.utils; + +public class RequestUriUtils { + + public static boolean isStaticResource(String requestURI) { + + return requestURI.startsWith("/css/") + || requestURI.startsWith("/js/") + || requestURI.startsWith("/images/") + || requestURI.startsWith("/public/") + || requestURI.startsWith("/pdfjs/") + || requestURI.endsWith(".svg"); + + } + +} diff --git a/src/main/resources/banner.txt b/src/main/resources/banner.txt new file mode 100644 index 00000000..e3188c2a --- /dev/null +++ b/src/main/resources/banner.txt @@ -0,0 +1,6 @@ + ____ _____ ___ ____ _ ___ _ _ ____ ____ ____ _____ +/ ___|_ _|_ _| _ \| | |_ _| \ | |/ ___| | _ \| _ \| ___| +\___ \ | | | || |_) | | | || \| | | _ _____| |_) | | | | |_ + ___) || | | || _ <| |___ | || |\ | |_| |_____| __/| |_| | _| +|____/ |_| |___|_| \_\_____|___|_| \_|\____| |_| |____/|_| +Powered by Spring Boot ${spring-boot.version} \ No newline at end of file diff --git a/src/main/resources/messages_ar_AR.properties b/src/main/resources/messages_ar_AR.properties index 90643fad..a8291de9 100644 --- a/src/main/resources/messages_ar_AR.properties +++ b/src/main/resources/messages_ar_AR.properties @@ -119,6 +119,7 @@ adminUserSettings.role=Role adminUserSettings.actions=Actions adminUserSettings.apiUser=Limited API User adminUserSettings.webOnlyUser=Web Only User +adminUserSettings.demoUser=Demo User (No custom settings) adminUserSettings.forceChange=Force user to change username/password on login adminUserSettings.submit=Save User diff --git a/src/main/resources/messages_en_GB.properties b/src/main/resources/messages_en_GB.properties index 2148629e..3a21736d 100644 --- a/src/main/resources/messages_en_GB.properties +++ b/src/main/resources/messages_en_GB.properties @@ -119,6 +119,7 @@ adminUserSettings.role=Role adminUserSettings.actions=Actions adminUserSettings.apiUser=Limited API User adminUserSettings.webOnlyUser=Web Only User +adminUserSettings.demoUser=Demo User (No custom settings) adminUserSettings.forceChange = Force user to change username/password on login adminUserSettings.submit=Save User diff --git a/src/main/resources/settings.yml.template b/src/main/resources/settings.yml.template index 0f229ba4..52d5e4de 100644 --- a/src/main/resources/settings.yml.template +++ b/src/main/resources/settings.yml.template @@ -5,10 +5,13 @@ security: enableLogin: false # set to 'true' to enable login csrfDisabled: true - + loginAttemptCount: 5 # lock user account after 5 tries + loginResetTimeMinutes : 120 # lock account for 2 hours after x attempts + system: defaultLocale: 'en-US' # Set the default language (e.g. 'de-DE', 'fr-FR', etc) googlevisibility: false # 'true' to allow Google visibility (via robots.txt), 'false' to disallow + enableAlphaFunctionality: false # Set to enable functionality which might need more testing before it fully goes live (This feature might make no changes) #ui: # appName: exampleAppName # Application's visible name diff --git a/src/main/resources/static/css/dark-mode.css b/src/main/resources/static/css/dark-mode.css index 82e01676..2f070c9c 100644 --- a/src/main/resources/static/css/dark-mode.css +++ b/src/main/resources/static/css/dark-mode.css @@ -75,6 +75,13 @@ table th, table td { border: none; color: #fff !important; } + +.btn-warning { + background-color: #ffc107 !important; + border: none; + color: #000 !important; +} + .btn-outline-secondary { color: #fff !important; border-color: #fff; @@ -92,6 +99,11 @@ hr { background-color: rgba(255, 255, 255, 0.6); /* for some browsers that might use background instead of border for
*/ } +.modal-content { + color: #fff !important; + border-color: #fff; +} + #global-buttons-container input { background-color: #323948; caret-color: #ffffff; diff --git a/src/main/resources/static/css/home.css b/src/main/resources/static/css/home.css index ab3b3348..fe184637 100644 --- a/src/main/resources/static/css/home.css +++ b/src/main/resources/static/css/home.css @@ -1,5 +1,5 @@ #searchBar { - background-image: url('/images/search.svg'); + background-image: url('../images/search.svg'); background-position: 16px 16px; background-repeat: no-repeat; width: 100%; diff --git a/src/main/resources/static/js/pipeline.js b/src/main/resources/static/js/pipeline.js index 06810743..4fcde3a0 100644 --- a/src/main/resources/static/js/pipeline.js +++ b/src/main/resources/static/js/pipeline.js @@ -12,20 +12,17 @@ function validatePipeline() { if (currentOperation === '/add-password') { containsAddPassword = true; } - console.log(currentOperation); - console.log(apiDocs[currentOperation]); + let currentOperationDescription = apiDocs[currentOperation]?.post?.description || ""; let nextOperationDescription = apiDocs[nextOperation]?.post?.description || ""; - console.log("currentOperationDescription", currentOperationDescription); - console.log("nextOperationDescription", nextOperationDescription); - + // Strip off 'ZIP-' prefix + currentOperationDescription = currentOperationDescription.replace("ZIP-", ''); + nextOperationDescription = nextOperationDescription.replace("ZIP-", ''); + let currentOperationOutput = currentOperationDescription.match(/Output:([A-Z\/]*)/)?.[1] || ""; let nextOperationInput = nextOperationDescription.match(/Input:([A-Z\/]*)/)?.[1] || ""; - console.log("Operation " + currentOperation + " Output: " + currentOperationOutput); - console.log("Operation " + nextOperation + " Input: " + nextOperationInput); - // Splitting in case of multiple possible output/input let currentOperationOutputArr = currentOperationOutput.split('/'); let nextOperationInputArr = nextOperationInput.split('/'); @@ -35,6 +32,7 @@ function validatePipeline() { console.log(`Intersection: ${intersection}`); if (intersection.length === 0) { + updateValidateButton(false); isValid = false; console.log(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); alert(`Incompatible operations: The output of operation '${currentOperation}' (${currentOperationOutput}) is not compatible with the input of the following operation '${nextOperation}' (${nextOperationInput}).`); @@ -43,6 +41,7 @@ function validatePipeline() { } } if (containsAddPassword && pipelineListItems[pipelineListItems.length - 1].querySelector('.operationName').textContent !== '/add-password') { + updateValidateButton(false); alert('The "add-password" operation should be at the end of the operations sequence. Please adjust the operations order.'); return false; } @@ -53,10 +52,20 @@ function validatePipeline() { console.error('Pipeline is not valid'); // Stop operation, maybe display an error to the user } - + updateValidateButton(isValid); return isValid; } +function updateValidateButton(isValid) { + var validateButton = document.getElementById('validateButton'); + if (isValid) { + validateButton.classList.remove('btn-danger'); + validateButton.classList.add('btn-success'); + } else { + validateButton.classList.remove('btn-success'); + validateButton.classList.add('btn-danger'); + } +} @@ -67,14 +76,14 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() return; } let selectedOperation = document.getElementById('operationsDropdown').value; - let parameters = operationSettings[selectedOperation] || {}; + + + var pipelineName = document.getElementById('pipelineName').value; + let pipelineList = document.getElementById('pipelineList').children; let pipelineConfig = { - "name": "uniquePipelineName", - "pipeline": [{ - "operation": selectedOperation, - "parameters": parameters - }], + "name": pipelineName, + "pipeline": [], "_examples": { "outputDir": "{outputFolder}/{folderName}", "outputFileName": "{filename}-{pipelineName}-{date}-{time}" @@ -83,6 +92,28 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() "outputFileName": "{filename}" }; + for (let i = 0; i < pipelineList.length; i++) { + let operationName = pipelineList[i].querySelector('.operationName').textContent; + let parameters = operationSettings[operationName] || {}; + + pipelineConfig.pipeline.push({ + "operation": operationName, + "parameters": parameters + }); + } + + + + + + + + + + + + + let pipelineConfigJson = JSON.stringify(pipelineConfig, null, 2); let formData = new FormData(); @@ -99,37 +130,50 @@ document.getElementById('submitConfigBtn').addEventListener('click', function() formData.append('json', pipelineConfigJson); console.log("formData", formData); - fetch('/handleData', { - method: 'POST', - body: formData + fetch('api/v1/pipeline/handleData', { + method: 'POST', + body: formData }) - .then(response => response.blob()) - .then(blob => { + .then(response => { + // Save the response to use it later + const responseToUseLater = response; + + return response.blob().then(blob => { + let url = window.URL.createObjectURL(blob); + let a = document.createElement('a'); + a.href = url; + + // Use responseToUseLater instead of response + const contentDisposition = responseToUseLater.headers.get('Content-Disposition'); + let filename = 'download'; + if (contentDisposition && contentDisposition.indexOf('attachment') !== -1) { + filename = decodeURIComponent(contentDisposition.split('filename=')[1].replace(/"/g, '')).trim(); + } + a.download = filename; + + document.body.appendChild(a); + a.click(); + a.remove(); + }); + }) + .catch((error) => { + console.error('Error:', error); + }); - let url = window.URL.createObjectURL(blob); - let a = document.createElement('a'); - a.href = url; - a.download = 'outputfile'; - document.body.appendChild(a); - a.click(); - a.remove(); - }) - .catch((error) => { - console.error('Error:', error); - }); }); let apiDocs = {}; - +let apiSchemas = {}; let operationSettings = {}; -fetch('v3/api-docs') +fetch('v1/api-docs') .then(response => response.json()) .then(data => { apiDocs = data.paths; + apiSchemas = data.components.schemas; let operationsDropdown = document.getElementById('operationsDropdown'); - const ignoreOperations = ["/handleData", "operationToIgnore"]; // Add the operations you want to ignore here + const ignoreOperations = ["/api/v1/pipeline/handleData", "/api/v1/pipeline/operationToIgnore"]; // Add the operations you want to ignore here operationsDropdown.innerHTML = ''; @@ -138,6 +182,9 @@ fetch('v3/api-docs') // Group operations by tags Object.keys(data.paths).forEach(operationPath => { let operation = data.paths[operationPath].post; + if(!operation || !operation.description) { + console.log(operationPath); + } if (operation && !ignoreOperations.includes(operationPath) && !operation.description.includes("Type:MISO")) { let operationTag = operation.tags[0]; // This assumes each operation has exactly one tag if (!operationsByTag[operationTag]) { @@ -146,9 +193,8 @@ fetch('v3/api-docs') operationsByTag[operationTag].push(operationPath); } }); - // Specify the order of tags - let tagOrder = ["General", "Security", "Convert", "Other", "Filter"]; + let tagOrder = ["General", "Security", "Convert", "Misc", "Filter"]; // Create dropdown options tagOrder.forEach(tag => { @@ -158,8 +204,18 @@ fetch('v3/api-docs') operationsByTag[tag].forEach(operationPath => { let option = document.createElement('option'); - let operationWithoutSlash = operationPath.replace(/\//g, ''); // Remove slashes - option.textContent = operationWithoutSlash; + + let operationPathDisplay = operationPath + operationPathDisplay = operationPath.replace(new RegExp("api/v1/" + tag.toLowerCase() + "/", 'i'), ""); + + + if(operationPath.includes("/convert")){ + operationPathDisplay = operationPathDisplay.replace(/^\//, '').replaceAll("/", " to "); + } else { + operationPathDisplay = operationPathDisplay.replace(/\//g, ''); // Remove slashes + } + operationPathDisplay = operationPathDisplay.replaceAll(" ","-"); + option.textContent = operationPathDisplay; option.value = operationPath; // Keep the value with slashes for querying group.appendChild(option); }); @@ -176,25 +232,40 @@ document.getElementById('addOperationBtn').addEventListener('click', function() let listItem = document.createElement('li'); listItem.className = "list-group-item"; - let hasSettings = (apiDocs[selectedOperation] && apiDocs[selectedOperation].post && - ((apiDocs[selectedOperation].post.parameters && apiDocs[selectedOperation].post.parameters.length > 0) || - (apiDocs[selectedOperation].post.requestBody && - apiDocs[selectedOperation].post.requestBody.content['multipart/form-data'].schema.properties))); + let hasSettings = false; + if (apiDocs[selectedOperation] && apiDocs[selectedOperation].post) { + const postMethod = apiDocs[selectedOperation].post; + + // Check if parameters exist + if (postMethod.parameters && postMethod.parameters.length > 0) { + hasSettings = true; + } else if (postMethod.requestBody && postMethod.requestBody.content['multipart/form-data']) { + // Extract the reference key + const refKey = postMethod.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); + // Check if the referenced schema exists and has properties + if (apiSchemas[refKey] && Object.keys(apiSchemas[refKey].properties).length > 0) { + hasSettings = true; + } + } + } listItem.innerHTML = ` -
-
${selectedOperation}
-
- - - - -
-
- `; +
+
${selectedOperation}
+
+ + + + +
+
+`; + pipelineList.appendChild(listItem); @@ -215,23 +286,28 @@ document.getElementById('addOperationBtn').addEventListener('click', function() listItem.querySelector('.remove').addEventListener('click', function(event) { event.preventDefault(); pipelineList.removeChild(listItem); + hideOrShowPipelineHeader(); }); listItem.querySelector('.pipelineSettings').addEventListener('click', function(event) { event.preventDefault(); showpipelineSettingsModal(selectedOperation); + hideOrShowPipelineHeader(); }); function showpipelineSettingsModal(operation) { let pipelineSettingsModal = document.getElementById('pipelineSettingsModal'); let pipelineSettingsContent = document.getElementById('pipelineSettingsContent'); let operationData = apiDocs[operation].post.parameters || []; - let requestBodyData = apiDocs[operation].post.requestBody.content['multipart/form-data'].schema.properties || {}; + // Resolve the $ref reference to get actual schema properties + let refKey = apiDocs[operation].post.requestBody.content['multipart/form-data'].schema['$ref'].split('/').pop(); + let requestBodyData = apiSchemas[refKey].properties || {}; + // Combine operationData and requestBodyData into a single array operationData = operationData.concat(Object.keys(requestBodyData).map(key => ({ - name: key, - schema: requestBodyData[key] + name: key, + schema: requestBodyData[key] }))); pipelineSettingsContent.innerHTML = ''; @@ -245,11 +321,15 @@ document.getElementById('addOperationBtn').addEventListener('click', function() let parameterLabel = document.createElement('label'); parameterLabel.textContent = `${parameter.name} (${parameter.schema.type}): `; - parameterLabel.title = parameter.description; + parameterLabel.title = parameter.schema.description; + parameterLabel.setAttribute('for', parameter.name); parameterDiv.appendChild(parameterLabel); + + let defaultValue = parameter.schema.example; + if (defaultValue === undefined) defaultValue = parameter.schema.default; let parameterInput; - + // check if enum exists in schema if (parameter.schema.enum) { // if enum exists, create a select element @@ -277,11 +357,12 @@ document.getElementById('addOperationBtn').addEventListener('click', function() parameterInput = document.createElement('input'); parameterInput.type = 'text'; parameterInput.className = "form-control"; - parameterInput.value = "automatedFileInput"; + parameterInput.value = "FileInputPathToBeInputtedManuallyOffline"; } else { parameterInput = document.createElement('input'); parameterInput.type = 'text'; parameterInput.className = "form-control"; + if (defaultValue !== undefined) parameterInput.value = defaultValue; } break; case 'number': @@ -289,10 +370,12 @@ document.getElementById('addOperationBtn').addEventListener('click', function() parameterInput = document.createElement('input'); parameterInput.type = 'number'; parameterInput.className = "form-control"; + if (defaultValue !== undefined) parameterInput.value = defaultValue; break; case 'boolean': parameterInput = document.createElement('input'); parameterInput.type = 'checkbox'; + if (defaultValue === true) parameterInput.checked = true; break; case 'array': case 'object': @@ -304,10 +387,13 @@ document.getElementById('addOperationBtn').addEventListener('click', function() parameterInput = document.createElement('input'); parameterInput.type = 'text'; parameterInput.className = "form-control"; + if (defaultValue !== undefined) parameterInput.value = defaultValue; } } parameterInput.id = parameter.name; + console.log("defaultValue", defaultValue); + console.log("parameterInput", parameterInput); if (operationSettings[operation] && operationSettings[operation][parameter.name] !== undefined) { let savedValue = operationSettings[operation][parameter.name]; @@ -327,7 +413,7 @@ document.getElementById('addOperationBtn').addEventListener('click', function() parameterInput.value = savedValue; } } - + console.log("parameterInput2", parameterInput); parameterDiv.appendChild(parameterInput); pipelineSettingsContent.appendChild(parameterDiv); @@ -340,50 +426,65 @@ document.getElementById('addOperationBtn').addEventListener('click', function() event.preventDefault(); let settings = {}; operationData.forEach(parameter => { - let value = document.getElementById(parameter.name).value; - switch (parameter.schema.type) { - case 'number': - case 'integer': - settings[parameter.name] = Number(value); - break; - case 'boolean': - settings[parameter.name] = document.getElementById(parameter.name).checked; - break; - case 'array': - case 'object': - try { - settings[parameter.name] = JSON.parse(value); - } catch (err) { - console.error(`Invalid JSON format for ${parameter.name}`); - } - break; - default: - settings[parameter.name] = value; + if(parameter.name !== "fileInput"){ + let value = document.getElementById(parameter.name).value; + switch (parameter.schema.type) { + case 'number': + case 'integer': + settings[parameter.name] = Number(value); + break; + case 'boolean': + settings[parameter.name] = document.getElementById(parameter.name).checked; + break; + case 'array': + case 'object': + try { + settings[parameter.name] = JSON.parse(value); + } catch (err) { + console.error(`Invalid JSON format for ${parameter.name}`); + } + break; + default: + settings[parameter.name] = value; + } } }); operationSettings[operation] = settings; - console.log(settings); - pipelineSettingsModal.style.display = "none"; + //pipelineSettingsModal.style.display = "none"; }); pipelineSettingsContent.appendChild(saveButton); - pipelineSettingsModal.style.display = "block"; + //pipelineSettingsModal.style.display = "block"; - pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { - pipelineSettingsModal.style.display = "none"; - } + //pipelineSettingsModal.getElementsByClassName("close")[0].onclick = function() { + // pipelineSettingsModal.style.display = "none"; + //} - window.onclick = function(event) { - if (event.target == pipelineSettingsModal) { - pipelineSettingsModal.style.display = "none"; - } - } + //window.onclick = function(event) { + // if (event.target == pipelineSettingsModal) { + // pipelineSettingsModal.style.display = "none"; + // } + //} } + hideOrShowPipelineHeader(); +}); + + + + var saveBtn = document.getElementById('savePipelineBtn'); - document.getElementById('savePipelineBtn').addEventListener('click', function() { + // Remove any existing event listeners + saveBtn.removeEventListener('click', savePipeline); + + // Add the event listener + saveBtn.addEventListener('click', savePipeline); + console.log("saveBtn", saveBtn) + function savePipeline() { + if (validatePipeline() === false) { return; } + var pipelineName = document.getElementById('pipelineName').value; let pipelineList = document.getElementById('pipelineList').children; let pipelineConfig = { @@ -393,31 +494,33 @@ document.getElementById('addOperationBtn').addEventListener('click', function() "outputDir": "{outputFolder}/{folderName}", "outputFileName": "{filename}-{pipelineName}-{date}-{time}" }, - "outputDir": "httpWebRequest", + "outputDir": "{outputFolder}", "outputFileName": "{filename}" }; for (let i = 0; i < pipelineList.length; i++) { let operationName = pipelineList[i].querySelector('.operationName').textContent; let parameters = operationSettings[operationName] || {}; - + + parameters['fileInput'] = 'automated'; + pipelineConfig.pipeline.push({ "operation": operationName, "parameters": parameters }); } - + console.log("Downloading.."); let a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([JSON.stringify(pipelineConfig, null, 2)], { type: 'application/json' })); - a.download = 'pipelineConfig.json'; + a.download = pipelineName + '.json'; a.style.display = 'none'; document.body.appendChild(a); a.click(); document.body.removeChild(a); - }); + } async function processPipelineConfig(configString) { let pipelineConfig = JSON.parse(configString); @@ -483,6 +586,7 @@ document.getElementById('addOperationBtn').addEventListener('click', function() processPipelineConfig(event.target.result); }; reader.readAsText(e.target.files[0]); + hideOrShowPipelineHeader(); }); document.getElementById('pipelineSelect').addEventListener('change', function(e) { @@ -491,4 +595,15 @@ document.getElementById('addOperationBtn').addEventListener('click', function() }); -}); \ No newline at end of file + function hideOrShowPipelineHeader() { + var pipelineHeader = document.getElementById('pipelineHeader'); + var pipelineList = document.getElementById('pipelineList'); + + if (pipelineList.children.length === 0) { + // Hide the pipeline header if there are no items in the pipeline list + pipelineHeader.style.display = 'none'; + } else { + // Show the pipeline header if there are items in the pipeline list + pipelineHeader.style.display = 'block'; + } + } diff --git a/src/main/resources/templates/account.html b/src/main/resources/templates/account.html index 83303057..c0e7a757 100644 --- a/src/main/resources/templates/account.html +++ b/src/main/resources/templates/account.html @@ -306,7 +306,7 @@
- + diff --git a/src/main/resources/templates/addUsers.html b/src/main/resources/templates/addUsers.html index 4ab59a8b..27701ae4 100644 --- a/src/main/resources/templates/addUsers.html +++ b/src/main/resources/templates/addUsers.html @@ -62,6 +62,7 @@ +
diff --git a/src/main/resources/templates/convert/pdf-to-csv.html b/src/main/resources/templates/convert/pdf-to-csv.html index d0ff04c8..5cfceeb6 100644 --- a/src/main/resources/templates/convert/pdf-to-csv.html +++ b/src/main/resources/templates/convert/pdf-to-csv.html @@ -13,7 +13,7 @@

-
+
diff --git a/src/main/resources/templates/fragments/navbar.html b/src/main/resources/templates/fragments/navbar.html index 31519548..2ddb3679 100644 --- a/src/main/resources/templates/fragments/navbar.html +++ b/src/main/resources/templates/fragments/navbar.html @@ -32,12 +32,12 @@ - +
  • @@ -185,7 +185,7 @@
    - + diff --git a/src/main/resources/templates/home.html b/src/main/resources/templates/home.html index 6189a960..946cdd9c 100644 --- a/src/main/resources/templates/home.html +++ b/src/main/resources/templates/home.html @@ -27,9 +27,10 @@
    - - - + +
    +     +
    diff --git a/src/main/resources/templates/pipeline.html b/src/main/resources/templates/pipeline.html index fa26ff29..51caf186 100644 --- a/src/main/resources/templates/pipeline.html +++ b/src/main/resources/templates/pipeline.html @@ -35,7 +35,8 @@

    - + +

    (Alpha) Pipeline Menu (Huge work in progress, very buggy!)