kamp/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2024-11-23 15:21:25 +01:00
Code

Extract images enhancements (#1757)

Browse Source 
* fix

* extarct images

* langs

* logging

* cuke fix

---------

Co-authored-by: a <a>
This commit is contained in:
Anthony Stirling 2024-08-27 11:46:18 +02:00 committed by GitHub
parent 63dfcfe688
commit 47314a0f38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 131 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
cucumber/exampleFiles/images.pdf
View File

Binary file not shown.

6
cucumber/features/general.feature
View File

@ -95,7 +95,7 @@ Feature: API Validation
@extract-images
Scenario Outline: Extract Image Scans
Scenario Outline: Extract Image Scans duplicates
Given I use an example file at "exampleFiles/images.pdf" as parameter "fileInput"
And the request data includes
| parameter | value |
@ -103,7 +103,7 @@ Feature: API Validation
When I send the API request to the endpoint "/api/v1/misc/extract-images"
Then the response content type should be "application/octet-stream"
And the response file should have extension ".zip"
And the response ZIP should contain 20 files
And the response ZIP should contain 2 files
And the response file should have size greater than 0
And the response status code should be 200
@ -112,5 +112,3 @@ Feature: API Validation
| png |
| gif |
| jpeg |

52
src/main/java/stirling/software/SPDF/controller/api/misc/ExtractImagesController.java
View File

@ -5,6 +5,9 @@ import java.awt.image.BufferedImage;
import java.awt.image.RenderedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@ -36,7 +39,8 @@ import io.github.pixee.security.Filenames;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import stirling.software.SPDF.model.api.PDFWithImageFormatRequest;
import stirling.software.SPDF.model.api.PDFExtractImagesRequest;
import stirling.software.SPDF.utils.ImageProcessingUtils;
import stirling.software.SPDF.utils.WebResponseUtils;
@RestController
@ -51,11 +55,11 @@ public class ExtractImagesController {
summary = "Extract images from a PDF file",
description =
"This endpoint extracts images from a given PDF file and returns them in a zip file. Users can specify the output image format. Input: PDF Output: IMAGE/ZIP Type: SIMO")
public ResponseEntity<byte[]> extractImages(@ModelAttribute PDFWithImageFormatRequest request)
public ResponseEntity<byte[]> extractImages(@ModelAttribute PDFExtractImagesRequest request)
throws IOException, InterruptedException, ExecutionException {
MultipartFile file = request.getFileInput();
String format = request.getFormat();
boolean allowDuplicates = request.isAllowDuplicates();
System.out.println(
System.currentTimeMillis() + " file=" + file.getName() + ", format=" + format);
PDDocument document = Loader.loadPDF(file.getBytes());
@ -75,7 +79,7 @@ public class ExtractImagesController {
String filename =
Filenames.toSimpleFileName(file.getOriginalFilename())
.replaceFirst("[.][^.]+$", "");
Set<Integer> processedImages = new HashSet<>();
Set<byte[]> processedImages = new HashSet<>();
if (useMultithreading) {
// Executor service to handle multithreading
@ -92,7 +96,13 @@ public class ExtractImagesController {
executor.submit(
() -> {
extractImagesFromPage(
page, format, filename, pageNum, processedImages, zos);
page,
format,
filename,
pageNum,
processedImages,
zos,
allowDuplicates);
return null;
});
@ -110,7 +120,8 @@ public class ExtractImagesController {
// Single-threaded extraction
for (int pgNum = 0; pgNum < document.getPages().getCount(); pgNum++) {
PDPage page = document.getPage(pgNum);
extractImagesFromPage(page, format, filename, pgNum + 1, processedImages, zos);
extractImagesFromPage(
page, format, filename, pgNum + 1, processedImages, zos, allowDuplicates);
}
}
@ -137,21 +148,34 @@ public class ExtractImagesController {
String format,
String filename,
int pageNum,
Set<Integer> processedImages,
ZipOutputStream zos)
Set<byte[]> processedImages,
ZipOutputStream zos,
boolean allowDuplicates)
throws IOException {
MessageDigest md;
try {
md = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
logger.error("MD5 algorithm not available for extractImages hash.", e);
return;
}
if (page.getResources() == null || page.getResources().getXObjectNames() == null) {
return;
}
int count = 1;
for (COSName name : page.getResources().getXObjectNames()) {
if (page.getResources().isImageXObject(name)) {
PDImageXObject image = (PDImageXObject) page.getResources().getXObject(name);
int imageHash = image.hashCode();
synchronized (processedImages) {
if (processedImages.contains(imageHash)) {
continue; // Skip already processed images
if (!allowDuplicates) {
byte[] data = ImageProcessingUtils.getImageData(image.getImage());
byte[] imageHash = md.digest(data);
synchronized (processedImages) {
if (processedImages.stream()
.anyMatch(hash -> Arrays.equals(hash, imageHash))) {
continue; // Skip already processed images
}
processedImages.add(imageHash);
}
processedImages.add(imageHash);
}
RenderedImage renderedImage = image.getImage();
@ -160,7 +184,7 @@ public class ExtractImagesController {
BufferedImage bufferedImage = convertToRGB(renderedImage, format);
// Write image to zip file
String imageName = filename + "_" + imageHash + " (Page " + pageNum + ")." + format;
String imageName = filename + "_page_" + pageNum + "_" + count++ + "." + format;
synchronized (zos) {
zos.putNextEntry(new ZipEntry(imageName));
ByteArrayOutputStream imageBaos = new ByteArrayOutputStream();

16
src/main/java/stirling/software/SPDF/model/api/PDFExtractImagesRequest.java Normal file
View File

@ -0,0 +1,16 @@
package stirling.software.SPDF.model.api;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@EqualsAndHashCode(callSuper = true)
public class PDFExtractImagesRequest extends PDFWithImageFormatRequest {
@Schema(
description =
"Boolean to enable/disable the saving of duplicate images, true to enable duplicates")
private boolean allowDuplicates;
}

1
src/main/java/stirling/software/SPDF/utils/GeneralUtils.java
View File

@ -262,4 +262,5 @@ public class GeneralUtils {
}
return true;
}
}

30
src/main/java/stirling/software/SPDF/utils/ImageProcessingUtils.java
View File

@ -1,6 +1,10 @@
package stirling.software.SPDF.utils;
import java.awt.image.BufferedImage;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.DataBufferInt;
import java.nio.ByteBuffer;
public class ImageProcessingUtils {
@ -29,4 +33,30 @@ public class ImageProcessingUtils {
}
return convertedImage;
}
public static byte[] getImageData(BufferedImage image) {
DataBuffer dataBuffer = image.getRaster().getDataBuffer();
if (dataBuffer instanceof DataBufferByte) {
return ((DataBufferByte) dataBuffer).getData();
} else if (dataBuffer instanceof DataBufferInt) {
int[] intData = ((DataBufferInt) dataBuffer).getData();
ByteBuffer byteBuffer = ByteBuffer.allocate(intData.length * 4);
byteBuffer.asIntBuffer().put(intData);
return byteBuffer.array();
} else {
int width = image.getWidth();
int height = image.getHeight();
byte[] data = new byte[width * height * 3];
int index = 0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int rgb = image.getRGB(x, y);
data[index++] = (byte) ((rgb >> 16) & 0xFF); // Red
data[index++] = (byte) ((rgb >> 8) & 0xFF); // Green
data[index++] = (byte) (rgb & 0xFF); // Blue
}
}
return data;
}
}
}

1
src/main/resources/messages_ar_AR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=معالجة PDF باستخدام OCR
extractImages.title=استخراج الصور
extractImages.header=استخراج الصور
extractImages.selectText=حدد تنسيق الصورة لتحويل الصور المستخرجة إلى
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=استخراج

1
src/main/resources/messages_bg_BG.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Обработка на PDF чрез OCR
extractImages.title=Извличане на изображения
extractImages.header=Извличане на изображения
extractImages.selectText=Изберете формат на изображението, в който да преобразувате извлечените изображения
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Извличане

1
src/main/resources/messages_ca_CA.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Processa PDF amb OCR
extractImages.title=Extreu Imatges
extractImages.header=Extreu Imatges
extractImages.selectText=Selecciona el format d'imatge al qual convertir les imatges extretes
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extreu

1
src/main/resources/messages_cs_CZ.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Zpracovat PDF s OCR
extractImages.title=Extrahovat obrázky
extractImages.header=Extrahovat obrázky
extractImages.selectText=Vyberte formát obrázku pro extrahované obrázky
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrahovat

3
src/main/resources/messages_da_DK.properties
View File

@ -1,7 +1,7 @@
###########
# Generic #
###########
# the direction that the language is written (ltr = left to right, rtl = right to left)
# the direction that the language is written (ltr=left to right, rtl = right to left)
language.direction=ltr
pdfPrompt=Vælg PDF-fil(er)
@ -802,6 +802,7 @@ ocr.submit=Process PDF with OCR
extractImages.title=Extract Images
extractImages.header=Extract Images
extractImages.selectText=Select image format to convert extracted images to
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extract

3
src/main/resources/messages_de_DE.properties
View File

@ -4,7 +4,7 @@
# the direction that the language is written (ltr=left to right, rtl = right to left)
language.direction=ltr
pdfPrompwt=PDF auswählen
pdfPrompt=Select PDF(s)
multiPdfPrompt=PDFs auswählen(2+)
multiPdfDropPrompt=Wählen Sie alle gewünschten PDFs aus (oder ziehen Sie sie per Drag & Drop hierhin)
imgPrompt=Wählen Sie ein Bild
@ -802,6 +802,7 @@ ocr.submit=PDF mit OCR verarbeiten
extractImages.title=Bilder extrahieren
extractImages.header=Bilder extrahieren
extractImages.selectText=Wählen Sie das Bildformat aus, in das extrahierte Bilder konvertiert werden sollen
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrahieren

1
src/main/resources/messages_el_GR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Επεξεργασία PDF με OCR
extractImages.title=Εξαγωγή Εικόνων
extractImages.header=Εξαγωγή Εικόνων
extractImages.selectText=Επιλέξτε μορφή εικόνας για να μετατρέψετε τις εξαγόμενες εικόνες
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Εξαγωγή

1
src/main/resources/messages_en_GB.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Process PDF with OCR
extractImages.title=Extract Images
extractImages.header=Extract Images
extractImages.selectText=Select image format to convert extracted images to
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extract

1
src/main/resources/messages_en_US.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Process PDF with OCR
extractImages.title=Extract Images
extractImages.header=Extract Images
extractImages.selectText=Select image format to convert extracted images to
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extract

1
src/main/resources/messages_es_ES.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Procesar PDF con OCR
extractImages.title=Extraer imágenes
extractImages.header=Extraer imágenes
extractImages.selectText=Seleccionar el formato de imagen para convertir las imágenes extraídas
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extraer

1
src/main/resources/messages_eu_ES.properties
View File

@ -802,6 +802,7 @@ ocr.submit=PDF prozesatu OCR-rekin
extractImages.title=Atera irudiak
extractImages.header=Atera irudiak
extractImages.selectText=Hautatu irudi-formatua ateratako irudiak bihurtzeko
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Atera

1
src/main/resources/messages_fr_FR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Traiter
extractImages.title=Extraire les images
extractImages.header=Extraire les images
extractImages.selectText=Format d’image dans lequel convertir les images extraites
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extraire

3
src/main/resources/messages_ga_IE.properties
View File

@ -1,7 +1,7 @@
###########
# Generic #
###########
# the direction that the language is written (ltr = left to right, rtl = right to left)
# the direction that the language is written (ltr=left to right, rtl = right to left)
language.direction=ltr
pdfPrompt=Roghnaigh PDF(s)
@ -802,6 +802,7 @@ ocr.submit=Próiseáil PDF le OCR
extractImages.title=Sliocht Íomhánna
extractImages.header=Sliocht Íomhánna
extractImages.selectText=Roghnaigh formáid íomhá chun íomhánna bainte a thiontú go
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Sliocht

1
src/main/resources/messages_hi_IN.properties
View File

@ -802,6 +802,7 @@ ocr.submit=OCR के साथ PDF प्रोसेस करें
extractImages.title=छवियां निकालें
extractImages.header=छवियां निकालें
extractImages.selectText=निकाली गई छवियों को कन्वर्ट करने के लिए छवि प्रारूप चुनें
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=निकालें

1
src/main/resources/messages_hr_HR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Obradi PDF sa OCR-om
extractImages.title=Ekstrakt slika
extractImages.header=Ekstrakt slika
extractImages.selectText=Odaberite format slike za pretvaranje izdvojenih slika
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Izdvajanje

1
src/main/resources/messages_hu_HU.properties
View File

@ -802,6 +802,7 @@ ocr.submit=PDF feldolgozása OCR-rel
extractImages.title=Képek kinyerése
extractImages.header=Képek kinyerése
extractImages.selectText=Válassza ki a képformátumot a kinyert képek konvertálásához
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Kinyerés

1
src/main/resources/messages_id_ID.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Memproses PDF dengan OCR
extractImages.title=Ekstrak Gambar
extractImages.header=Mengekstrak Gambar
extractImages.selectText=Pilih format gambar yang akan dikonversi
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Ekstrak

1
src/main/resources/messages_it_IT.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Scansiona testo nel PDF con OCR
extractImages.title=Estrai immagini
extractImages.header=Estrai immagini
extractImages.selectText=Seleziona il formato in cui salvare le immagini estratte
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Estrai

1
src/main/resources/messages_ja_JP.properties
View File

@ -802,6 +802,7 @@ ocr.submit=OCRでPDFを処理する
extractImages.title=画像の抽出
extractImages.header=画像の抽出
extractImages.selectText=抽出した画像のフォーマットを選択
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=抽出

1
src/main/resources/messages_ko_KR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=인식
extractImages.title=이미지 추출
extractImages.header=이미지 추출
extractImages.selectText=추출된 이미지를 변환할 이미지 형식을 선택합니다.
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=추출

1
src/main/resources/messages_nl_NL.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Verwerk PDF met OCR
extractImages.title=Afbeeldingen extraheren
extractImages.header=Afbeeldingen extraheren
extractImages.selectText=Selecteer het beeldformaat voor geëxtraheerde afbeeldingen
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extraheer

1
src/main/resources/messages_no_NB.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Behandle PDF med OCR
extractImages.title=Hent ut bilder
extractImages.header=Hent ut bilder
extractImages.selectText=Velg bildeformat for å konvertere de hentede bildene til
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Hent ut

1
src/main/resources/messages_pl_PL.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Przetwarzaj PDF za pomocą OCR
extractImages.title=Wyodrębnij obrazy
extractImages.header=Wyodrębnij obrazy
extractImages.selectText=Wybierz format obrazu, na który chcesz przekonwertować wyodrębniony obraz.
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Wyodrębnij

1
src/main/resources/messages_pt_BR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Processar PDF com OCR
extractImages.title=Extrair imagens
extractImages.header=Extrair imagens
extractImages.selectText=Selecione o formato de imagem para converter as imagens extraídas
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrair

1
src/main/resources/messages_pt_PT.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Processar PDF com OCR
extractImages.title=Extrair Imagens
extractImages.header=Extrair Imagens
extractImages.selectText=Selecione o formato de imagem para converter as imagens extraídas
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrair

1
src/main/resources/messages_ro_RO.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Procesează PDF-ul cu OCR
extractImages.title=Extrage Imagini
extractImages.header=Extrage Imagini
extractImages.selectText=Selectați formatul imaginii în care să se convertească imaginile extrase
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrage

1
src/main/resources/messages_ru_RU.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Обработка PDF с OCR
extractImages.title=Извлечь изображения
extractImages.header=Извлечь изображения
extractImages.selectText=Выберите формат изображения для преобразования извлеченных изображений в
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Извлечь

1
src/main/resources/messages_sk_SK.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Spracovať PDF s OCR
extractImages.title=Extrahovať obrázky
extractImages.header=Extrahovať obrázky
extractImages.selectText=Vyberte formát obrázka na konverziu extrahovaných obrázkov
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrahovať

1
src/main/resources/messages_sr_LATN_RS.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Obradi PDF sa OCR-om
extractImages.title=Izdvajanje slika
extractImages.header=Izdvajanje slika
extractImages.selectText=Odaberite format slike za konvertovanje izdvojenih slika
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Izdvajanje

1
src/main/resources/messages_sv_SE.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Bearbeta PDF med OCR
extractImages.title=Extrahera bilder
extractImages.header=Extrahera bilder
extractImages.selectText=Välj bildformat att konvertera extraherade bilder till
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Extrahera

1
src/main/resources/messages_th_TH.properties
View File

@ -802,6 +802,7 @@ ocr.submit=ประมวลผล PDF ด้วย OCR
extractImages.title=แยกรูปภาพ
extractImages.header=แยกรูปภาพ
extractImages.selectText=เลือกรูปแบบภาพที่จะใช้ในการแปลงรูปภาพที่แยกได้
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=แยก

1
src/main/resources/messages_tr_TR.properties
View File

@ -802,6 +802,7 @@ ocr.submit=PDF'i OCR(Metin Tanıma) ile İşle
extractImages.title=Resimleri Çıkar
extractImages.header=Resimleri Çıkar
extractImages.selectText=Çıkarılan resimleri dönüştürmek için resim formatını seçin
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Çıkar

1
src/main/resources/messages_uk_UA.properties
View File

@ -802,6 +802,7 @@ ocr.submit=Обробка PDF з OCR
extractImages.title=Витягнути зображення
extractImages.header=Витягнути зображення
extractImages.selectText=Виберіть формат зображення для перетворення витягнутих зображень у
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Витягнути

3
src/main/resources/messages_vi_VN.properties
View File

@ -1,7 +1,7 @@
###########
# Generic #
###########
# the direction that the language is written (ltr = left to right, rtl = right to left)
# the direction that the language is written (ltr=left to right, rtl = right to left)
language.direction=ltr
pdfPrompt=Chọn (các) tệp PDF
@ -802,6 +802,7 @@ ocr.submit=Xử lý PDF với OCR
extractImages.title=Trích xuất hình ảnh
extractImages.header=Trích xuất hình ảnh
extractImages.selectText=Chọn định dạng hình ảnh để chuyển đổi hình ảnh đã trích xuất
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=Trích xuất

1
src/main/resources/messages_zh_CN.properties
View File

@ -802,6 +802,7 @@ ocr.submit=用OCR处理PDF
extractImages.title=提取图像
extractImages.header=提取图像
extractImages.selectText=选择图像格式，将提取的图像转换为
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=提取

1
src/main/resources/messages_zh_TW.properties
View File

@ -802,6 +802,7 @@ ocr.submit=使用 OCR 處理 PDF
extractImages.title=提取圖片
extractImages.header=提取圖片
extractImages.selectText=選擇要轉換提取影像的影像格式
extractImages.allowDuplicates=Save duplicate images
extractImages.submit=提取

4
src/main/resources/templates/misc/extract-images.html
View File

@ -27,6 +27,10 @@
<option value="gif">GIF</option>
</select>
</div>
<div class="mb-3">
<input type="checkbox" name="allowDuplicates" id="allowDuplicates">
<label for="allowDuplicates" th:text="#{extractImages.allowDuplicates}"></label>
</div>
<button type="submit" id="submitBtn" class="btn btn-primary" th:text="#{extractImages.submit}"></button>
</form>
</div>