fix of the json upload format

- Created '/upload_csv_batch' endpoint to handle CSV parsing and multiple image uploads
- Added automatic WebP conversion and GridFS storage for batch images
- Implemented grouping logic via 'series_group_id' based on item name
- Created '/batch_upload' route and 'upload_batch.html' for a seamless async frontend

Web/app.py

@@ -108,7 +108,7 @@ app.config['UPLOAD_FOLDER'] = cfg.UPLOAD_FOLDER
 app.config['THUMBNAIL_FOLDER'] = cfg.THUMBNAIL_FOLDER
 app.config['PREVIEW_FOLDER'] = cfg.PREVIEW_FOLDER
 app.config['ALLOWED_EXTENSIONS'] = set(cfg.ALLOWED_EXTENSIONS)
-app.config['MAX_CONTENT_LENGTH'] = max(cfg.MAX_UPLOAD_MB, cfg.IMAGE_MAX_UPLOAD_MB, cfg.VIDEO_MAX_UPLOAD_MB) * 1024 * 1024
+app.config['MAX_CONTENT_LENGTH'] = 1024 * 1024 * 1024
 app.config['SESSION_COOKIE_HTTPONLY'] = True
 app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
 app.config['SESSION_COOKIE_SECURE'] = cfg.SSL_ENABLED if os.getenv('INVENTAR_SESSION_COOKIE_SECURE') is None else os.getenv('INVENTAR_SESSION_COOKIE_SECURE', '').strip().lower() in ('1', 'true', 'yes', 'on')
@@ -665,11 +665,14 @@ def handle_unexpected_exception(e):
 
 
 def _csrf_error_response(message='CSRF token fehlt oder ist ungültig.'):
-    if request.is_json or request.path.startswith('/api/') or request.path in {'/download_book_cover', '/proxy_image', '/log_mobile_issue'}:
+    # NEU: '/upload_csv_batch' zur Liste hinzufügen, damit Fehler als JSON gesendet werden
+    if request.is_json or request.path.startswith('/api/') or request.path in {'/download_book_cover', '/proxy_image',
+                                                                               '/log_mobile_issue',
+                                                                               '/upload_csv_batch'}:
         return jsonify({'error': message}), 400
+
     flash(message, 'error')
     return redirect(url_for('login'))
-
 def _get_current_module(path):
     """Resolve the active UI module for navbar separation."""
     mod = cfg.MODULES.get_module_for_path(path)
@@ -11845,3 +11848,267 @@ def test_push_notification():
     except Exception as e:
         app.logger.error(f'Error sending test push: {e}')
         return jsonify({'success': False}), 500
+
+
+@app.route('/batch_upload', methods=['GET'])
+def batch_upload_page():
+    """
+    Serves the HTML frontend for the batch CSV and image upload.
+    """
+    # Check permissions if necessary, similar to your other routes
+    if 'username' not in session:
+        flash('Bitte melden Sie sich an.', 'error')
+        return redirect(url_for('login'))
+
+    return render_template('upload_batch.html')
+
+
+def clean_db_field(val):
+    """Bereinigt Werte, die fälschlicherweise als String-Listen aus der CSV kommen."""
+    import pandas as pd
+    if not val or pd.isna(val):
+        return None
+
+    val_str = str(val).strip()
+
+    # Erkennt und entpackt String-Listen wie "['100465']" oder '["100465"]'
+    if (val_str.startswith("['") and val_str.endswith("']")) or (val_str.startswith('["') and val_str.endswith('"]')):
+        inner = val_str[2:-2].strip()
+        return inner if inner and inner != "''" and inner != '""' else None
+
+    if val_str in ("[]", "['']", '[""]', "nan", "None"):
+        return None
+
+    return val_str
+
+@app.route('/upload_csv_batch', methods=['POST'])
+def upload_csv_batch():
+    """
+    Route for batch adding new items to the inventory via CSV.
+    Handles CSV parsing, bulk image upload with deduplication (SHA-256 hash matching),
+    GridFS storage, code generation, location syncing, and grouped item creation.
+    """
+    import pandas as pd
+    import ast
+    import hashlib
+
+    username = session.get('username', 'System')
+
+    def generate_unique_batch_code(base_code, position):
+        if base_code:
+            return f"{base_code}-{position}"
+        else:
+            random_prefix = str(uuid.uuid4())[:6].upper()
+            return f"BATCH-{random_prefix}-{position}"
+
+    fs = get_gridfs()
+    upload_session_id = str(uuid.uuid4())[:8]
+    app.logger.info(f"Starting CSV Batch upload session {upload_session_id} - User: {username}")
+
+    # 1. Dateien aus dem Request empfangen
+    if 'csv_file' not in request.files:
+        return jsonify({"success": False, "message": "Keine CSV-Datei hochgeladen"}), 400
+
+    csv_file = request.files['csv_file']
+    uploaded_images = request.files.getlist('images')
+
+    # 2. CSV Einlesen und Validieren
+    try:
+        df = pd.read_csv(csv_file, sep=',')
+    except Exception as e:
+        app.logger.error(f"[Upload {upload_session_id}] Fehler beim Lesen der CSV: {str(e)}")
+        return jsonify({"success": False, "message": f"Fehler beim Lesen der CSV: {str(e)}"}), 400
+
+    if 'Name' not in df.columns:
+        return jsonify({"success": False, "message": "Die CSV muss zwingend eine 'Name' Spalte enthalten."}), 400
+
+    # 3. Bilder verarbeiten & Duplikate im selben Durchlauf filtern (Hash-Matching)
+    image_mapping = {}
+    processed_hashes = {}
+    processed_count = 0
+    dedup_count = 0
+    error_count = 0
+
+    for index, image in enumerate(uploaded_images):
+        if not image or not image.filename:
+            continue
+
+        original_secure_name = secure_filename(image.filename)
+        base_name_no_ext = os.path.splitext(original_secure_name)[0]
+        image_log_prefix = f"[Upload {upload_session_id}][Image {index + 1}/{len(uploaded_images)}]"
+
+        try:
+            image.seek(0)
+            image_bytes = image.read()
+            if not image_bytes:
+                error_count += 1
+                continue
+
+            img_hash = hashlib.sha256(image_bytes).hexdigest()
+
+            if img_hash in processed_hashes:
+                existing_filename = processed_hashes[img_hash]
+                image_mapping[base_name_no_ext] = existing_filename
+                dedup_count += 1
+                continue
+
+            optimized_io = io.BytesIO()
+            with Image.open(io.BytesIO(image_bytes)) as img:
+                if img.mode not in ('RGB', 'RGBA'):
+                    img = img.convert('RGBA')
+
+                max_width = 500
+                if img.width > max_width:
+                    ratio = max_width / img.width
+                    new_size = (max_width, int(img.height * ratio))
+                    img = img.resize(new_size, Image.Resampling.LANCZOS)
+
+                img.save(optimized_io, format='WEBP', quality=85, optimize=True)
+
+            optimized_io.seek(0)
+            new_filename = f"{uuid.uuid4().hex}_{int(time.time())}.webp"
+
+            fs.put(
+                optimized_io,
+                filename=new_filename,
+                content_type='image/webp',
+                metadata={
+                    'original_filename': original_secure_name,
+                    'upload_session': upload_session_id,
+                    'batch_upload': True
+                }
+            )
+
+            processed_hashes[img_hash] = new_filename
+            image_mapping[base_name_no_ext] = new_filename
+            processed_count += 1
+
+        except Exception as e:
+            app.logger.error(f"{image_log_prefix} Processing failed: {str(e)}")
+            error_count += 1
+
+    # 4. Predefined Locations laden
+    try:
+        predefined_locations = it.get_predefined_locations()
+    except Exception:
+        predefined_locations = []
+
+    # 5. Dataframe bereinigen
+    df['Name'] = df['Name'].fillna('Unbenannt').astype(str).str.strip()
+    df = df.fillna({
+        'Ort': 'Unbekannt',
+        'Beschreibung': '',
+        'Code_4': '',
+        'Anschaffungsjahr': '',
+        'Anschaffungskosten': ''
+    })
+
+    # --- WICHTIG: Gruppierung über einen normalisierten Schlüssel ermöglichen ---
+    # Erstellt eine unsichtbare Hilfsspalte, die Leerzeichen/Groß-Kleinschreibung ignoriert,
+    # damit identische Artikel-Typen sauber als Serie erkannt werden.
+    df['GroupKey'] = df['Name'].str.lower()
+
+    created_item_ids = []
+    grouped_items = df.groupby('GroupKey')
+
+    for group_key, group in grouped_items:
+        item_count = len(group)
+        series_group_id = str(uuid.uuid4()) if item_count > 1 else None
+        parent_item_id = None
+
+        # Originalen Namen des ersten Elements der Gruppe übernehmen
+        actual_group_name = group.iloc[0]['Name']
+
+        # Basis-Code für automatisierte Seriencodes ermitteln
+        first_row_code = clean_db_field(group.iloc[0].get('Code_4', ''))
+        base_code = first_row_code if first_row_code else None
+
+        for position, (index, row) in enumerate(group.iterrows(), start=1):
+
+            ort_val = str(row['Ort']).strip()
+            if ort_val and ort_val not in predefined_locations:
+                try:
+                    it.add_predefined_location(ort_val)
+                    predefined_locations.append(ort_val)
+                except Exception as e:
+                    app.logger.warning(f"Ort {ort_val} konnte nicht hinzugefügt werden: {e}")
+
+            # Bilder zuordnen und pro Artikel deduplizieren
+            item_image_filenames = []
+            if 'Images' in row and pd.notna(row['Images']):
+                try:
+                    img_list = ast.literal_eval(str(row['Images']))
+                    if isinstance(img_list, list):
+                        for img_name in img_list:
+                            base_img_name = os.path.splitext(img_name)[0]
+                            if base_img_name in image_mapping:
+                                item_image_filenames.append(image_mapping[base_img_name])
+                except (ValueError, SyntaxError):
+                    pass
+
+            unique_image_filenames = []
+            for img in item_image_filenames:
+                if img not in unique_image_filenames:
+                    unique_image_filenames.append(img)
+
+            def parse_filter_col(col_data):
+                try:
+                    res = ast.literal_eval(str(col_data))
+                    return res if isinstance(res, list) else []
+                except Exception:
+                    return []
+
+            filter_upload = parse_filter_col(row.get('Filter', '[]'))
+            filter_upload2 = parse_filter_col(row.get('Filter2', '[]'))
+            filter_upload3 = parse_filter_col(row.get('Filter3', '[]'))
+
+            reservierbar = bool(row.get('Reservierbar', False))
+
+            # Code_4 Behandlung
+            row_code = clean_db_field(row.get('Code_4', ''))
+            if row_code:
+                unique_code = row_code
+            elif item_count > 1:
+                unique_code = generate_unique_batch_code(base_code, position)
+            else:
+                unique_code = None
+
+            # DB Insert
+            item_id = it.add_item(
+                str(actual_group_name),                       # 1. Name
+                ort_val,                                      # 2. Ort
+                str(row['Beschreibung']),                     # 3. Beschreibung
+                unique_image_filenames,                       # 4. Image Filenames (GridFS)
+                filter_upload,                                # 5. Filter 1
+                filter_upload2,                               # 6. Filter 2
+                filter_upload3,                               # 7. Filter 3
+                clean_db_field(row.get('Anschaffungsjahr')),  # 8. Jahr
+                clean_db_field(row.get('Anschaffungskosten')),# 9. Kosten
+                unique_code,                                  # 10. Unique Code / Code_4
+                reservierbar=reservierbar,
+                series_group_id=series_group_id,
+                series_count=item_count,
+                series_position=position,
+                is_grouped_sub_item=(position > 1),
+                parent_item_id=parent_item_id,
+                isbn=str(row.get('ISBN', '')),
+                item_type=str(row.get('Item_Type', 'other')),
+                library_category=str(row.get('Library_Category', '')),
+                is_library=bool(row.get('Is_Library', False))
+            )
+
+            if item_id:
+                created_item_ids.append(item_id)
+                if position == 1:
+                    parent_item_id = str(item_id)
+            else:
+                app.logger.error(f"Fehler beim Erstellen von Item: {actual_group_name} (Index {index})")
+
+    return jsonify({
+        "success": True,
+        "message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert.",
+        "created_count": len(created_item_ids),
+        "images_processed": processed_count,
+        "images_deduplicated": dedup_count,
+        "images_failed": error_count
+    }), 200

Web/requirements.txt

@@ -17,4 +17,5 @@ cryptography>=42.0.0
 pywebpush
 py-vapid>=1.9.0
 beautifulsoup4
-pywebpush
+pywebpush
+pandas

Web/templates/upload_batch.html

@@ -0,0 +1,253 @@
+<!DOCTYPE html>
+<html lang="de">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Batch Upload</title>
+
+    <!-- CSRF-Token für JavaScript bereitstellen -->
+    <meta name="csrf-token" content="{{ session.get('_csrf_token', '') }}">
+
+    <style>
+        body { font-family: sans-serif; padding: 20px; background: #f4f7f6; }
+        .upload-container { background: white; padding: 20px; border-radius: 8px; max-width: 500px; margin: 0 auto; }
+        .form-group { margin-bottom: 15px; }
+        label { display: block; font-weight: bold; margin-bottom: 5px; }
+        input[type="file"] { width: 100%; padding: 8px; box-sizing: border-box; }
+        .btn-submit { width: 100%; padding: 10px; background: #4a90e2; color: white; border: none; border-radius: 4px; font-weight: bold; cursor: pointer; }
+        .btn-submit:disabled { background: #ccc; }
+        #uploadProgress { margin-top: 20px; display: none; }
+        progress { width: 100%; height: 20px; }
+        #logList { background: #fafafa; border: 1px solid #eee; padding: 10px; max-height: 150px; overflow-y: auto; font-size: 0.85em; list-style: none; }
+    </style>
+</head>
+<body>
+
+<div class="upload-container">
+    <h2>Inventar Batch Upload</h2>
+
+    <form id="batchUploadForm">
+        <div class="form-group">
+            <label for="csv_file">1. items.csv auswählen</label>
+            <input type="file" id="csv_file" name="csv_file" accept=".csv" required>
+        </div>
+
+        <div class="form-group">
+            <label for="images">2. Bilder auswählen</label>
+            <input type="file" id="images" name="images" accept="image/*" multiple required>
+        </div>
+
+        <button type="submit" id="uploadBtn" class="btn-submit">Daten hochladen</button>
+    </form>
+
+    <div id="uploadProgress">
+        <div id="progressText">Bereite Upload vor...</div>
+        <progress id="progressBar" value="0" max="100"></progress>
+        <ul id="logList"></ul>
+    </div>
+</div>
+
+<script>
+// 1. Ein echter CSV-Parser, der Zeilenumbrüche und Kommas in Texten korrekt ignoriert
+function parseCSV(csvString) {
+    const rows = [];
+    let currentRow = [];
+    let currentCell = '';
+    let insideQuotes = false;
+
+    for (let i = 0; i < csvString.length; i++) {
+        const char = csvString[i];
+        const nextChar = csvString[i + 1];
+
+        if (char === '"' && insideQuotes && nextChar === '"') {
+            currentCell += '"';
+            i++; // Escaped Quotes ("") überspringen
+        } else if (char === '"') {
+            insideQuotes = !insideQuotes;
+        } else if (char === ',' && !insideQuotes) {
+            currentRow.push(currentCell);
+            currentCell = '';
+        } else if ((char === '\n' || char === '\r') && !insideQuotes) {
+            if (char === '\r' && nextChar === '\n') i++; // Windows Umbrüche überspringen
+            currentRow.push(currentCell);
+            // Leere Zeilen ignorieren
+            if (currentRow.length > 1 || currentRow[0] !== '') {
+                rows.push(currentRow);
+            }
+            currentRow = [];
+            currentCell = '';
+        } else {
+            currentCell += char;
+        }
+    }
+    if (currentCell !== '' || currentRow.length > 0) {
+        currentRow.push(currentCell);
+        if (currentRow.length > 1 || currentRow[0] !== '') {
+            rows.push(currentRow);
+        }
+    }
+    return rows;
+}
+
+// 2. Baut das Array wieder sicher zu einer sauberen CSV-Zeile für das Backend zusammen
+function rowToCSV(rowArray) {
+    return rowArray.map(cell => {
+        if (cell === null || cell === undefined) return '';
+        let cellStr = String(cell);
+        // Wenn kritische Zeichen drin sind, sauber in Anführungszeichen verpacken
+        if (cellStr.includes(',') || cellStr.includes('"') || cellStr.includes('\n') || cellStr.includes('\r')) {
+            return '"' + cellStr.replace(/"/g, '""') + '"';
+        }
+        return cellStr;
+    }).join(',');
+}
+
+// 3. Bildnamen extrahieren (bleibt wie es war)
+function extractImageNames(cellValue) {
+    if (!cellValue) return [];
+    let raw = String(cellValue).replace(/^["']|["']$/g, '').trim();
+    if (raw.startsWith('[') && raw.endsWith(']')) {
+        try {
+            const jsonValid = raw.replace(/'/g, '"');
+            return JSON.parse(jsonValid);
+        } catch (e) {
+            const matches = raw.match(/['"]([^'"]+)['"]/g);
+            if (matches) return matches.map(m => m.replace(/['"]/g, ''));
+        }
+    }
+    return raw ? [raw] : [];
+}
+
+document.getElementById('batchUploadForm').addEventListener('submit', async function(e) {
+    e.preventDefault();
+
+    const csvInput = document.getElementById('csv_file');
+    const imageInput = document.getElementById('images');
+    const uploadBtn = document.getElementById('uploadBtn');
+    const progressContainer = document.getElementById('uploadProgress');
+    const progressBar = document.getElementById('progressBar');
+    const progressText = document.getElementById('progressText');
+    const logList = document.getElementById('logList');
+
+    const csrfToken = document.querySelector('meta[name="csrf-token"]')?.getAttribute('content');
+
+    if (!csvInput.files.length) {
+        alert("Bitte wähle eine CSV-Datei aus.");
+        return;
+    }
+
+    uploadBtn.disabled = true;
+    progressContainer.style.display = 'block';
+    logList.innerHTML = '';
+
+    const log = (msg) => {
+        const li = document.createElement('li');
+        li.textContent = msg;
+        logList.appendChild(li);
+        logList.scrollTop = logList.scrollHeight;
+    };
+
+    const csvFile = csvInput.files[0];
+    const allImages = Array.from(imageInput.files);
+
+    const imageMap = new Map();
+    allImages.forEach(file => {
+        imageMap.set(file.name.toLowerCase(), file);
+    });
+
+    const BATCH_SIZE = 50;
+
+    try {
+        const csvText = await csvFile.text();
+        // Hier rufen wir jetzt unseren sicheren Parser auf!
+        const allRows = parseCSV(csvText);
+
+        if (allRows.length <= 1) {
+            throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
+        }
+
+        const headerRow = allRows[0];
+        const dataRows = allRows.slice(1);
+
+        log(`${dataRows.length} Einträge gefunden. Bereite Batches vor...`);
+
+        // Spalten-Index von "Images" sicher ermitteln
+        const imagesColIndex = headerRow.findIndex(h => h.toLowerCase().trim() === 'images');
+
+        // In Batches aufteilen
+        const batches = [];
+        for (let i = 0; i < dataRows.length; i += BATCH_SIZE) {
+            batches.push(dataRows.slice(i, i + BATCH_SIZE));
+        }
+
+        progressBar.max = batches.length;
+        progressBar.value = 0;
+
+        for (let b = 0; b < batches.length; b++) {
+            const batchRows = batches[b];
+            progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
+
+            const requiredImagesForBatch = new Set();
+
+            if (imagesColIndex !== -1) {
+                batchRows.forEach(rowCols => {
+                    if (rowCols[imagesColIndex]) {
+                        const imgNames = extractImageNames(rowCols[imagesColIndex]);
+                        imgNames.forEach(name => {
+                            const fileMatch = imageMap.get(name.toLowerCase());
+                            if (fileMatch) {
+                                requiredImagesForBatch.add(fileMatch);
+                            }
+                        });
+                    }
+                });
+            }
+
+            // Sicherer Zusammenbau des CSV-Batches
+            const batchCsvArray = [headerRow, ...batchRows];
+            const batchCsvText = batchCsvArray.map(rowToCSV).join('\n');
+            const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
+
+            const formData = new FormData();
+            formData.append('csv_file', batchCsvBlob, `batch_${b + 1}.csv`);
+
+            if (csrfToken) {
+                formData.append('csrf_token', csrfToken);
+            }
+
+            requiredImagesForBatch.forEach(imgFile => {
+                formData.append('images', imgFile);
+            });
+
+            log(`Batch ${b + 1}: ${batchRows.length} Items & ${requiredImagesForBatch.size} zugehörige Bilder.`);
+
+            const response = await fetch('/upload_csv_batch', {
+                method: 'POST',
+                body: formData,
+                headers: {
+                    'X-CSRFToken': csrfToken || ''
+                }
+            });
+
+            const result = await response.json();
+
+            if (!response.ok || !result.success) {
+                throw new Error(result.message || `Server-Fehler ${response.status}`);
+            }
+
+            log(`Batch ${b + 1} abgeschlossen.`);
+            progressBar.value = b + 1;
+        }
+
+        progressText.textContent = "Upload erfolgreich beendet! Keine verschobenen Spalten mehr.";
+        uploadBtn.disabled = false;
+
+    } catch (err) {
+        alert("Upload abgebrochen: " + err.message);
+        log("Fehler: " + err.message);
+        uploadBtn.disabled = false;
+    }
+});
+</script>
+</body>
+</html>

requirements.txt

@@ -17,4 +17,5 @@ cryptography>=42.0.0
 pywebpush
 py-vapid>=1.9.0
 beautifulsoup4
-pywebpush
+pywebpush
+pandas