Compare commits

...

14 Commits

Author SHA1 Message Date
Aiirondev_dev 33c65f21b6 fix of the json upload format 2026-08-03 23:55:30 +02:00
Aiirondev_dev fd242a6a0a fix of the json upload format 2026-08-03 23:45:59 +02:00
Aiirondev_dev 2892024969 fix of the json upload format 2026-08-03 23:26:51 +02:00
Aiirondev_dev 27f5280bbf fix of the json upload format 2026-08-03 19:36:56 +02:00
Aiirondev_dev 82898e34cb fix of the json upload format 2026-08-03 19:16:32 +02:00
Aiirondev_dev 44392c2c31 fix of the json upload format 2026-08-03 18:51:47 +02:00
Aiirondev_dev 58d94b716f fix of the json upload format 2026-08-03 18:34:27 +02:00
Aiirondev_dev 579a0ddb75 fix of the json upload format 2026-08-03 18:22:27 +02:00
Aiirondev_dev 0f21e8d9ca fix of the json upload format 2026-08-03 01:02:40 +02:00
Aiirondev_dev 12f7240cd2 fix of the json upload format 2026-08-03 00:52:39 +02:00
Aiirondev_dev 54d8d61358 fix of the json upload format 2026-08-03 00:28:53 +02:00
Aiirondev_dev 5052dd9de6 fix of the json upload format 2026-08-03 00:20:53 +02:00
Aiirondev_dev bf31ee2d16 feat: add batch CSV and image upload logic
- Created '/upload_csv_batch' endpoint to handle CSV parsing and multiple image uploads
- Added automatic WebP conversion and GridFS storage for batch images
- Implemented grouping logic via 'series_group_id' based on item name
- Created '/batch_upload' route and 'upload_batch.html' for a seamless async frontend
2026-08-03 00:06:48 +02:00
Aiirondev_dev b847930500 fix of the username decryption for the logs 2026-08-02 21:32:26 +02:00
5 changed files with 585 additions and 16 deletions
+271 -8
View File
@@ -108,7 +108,7 @@ app.config['UPLOAD_FOLDER'] = cfg.UPLOAD_FOLDER
app.config['THUMBNAIL_FOLDER'] = cfg.THUMBNAIL_FOLDER
app.config['PREVIEW_FOLDER'] = cfg.PREVIEW_FOLDER
app.config['ALLOWED_EXTENSIONS'] = set(cfg.ALLOWED_EXTENSIONS)
app.config['MAX_CONTENT_LENGTH'] = max(cfg.MAX_UPLOAD_MB, cfg.IMAGE_MAX_UPLOAD_MB, cfg.VIDEO_MAX_UPLOAD_MB) * 1024 * 1024
app.config['MAX_CONTENT_LENGTH'] = 1024 * 1024 * 1024
app.config['SESSION_COOKIE_HTTPONLY'] = True
app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
app.config['SESSION_COOKIE_SECURE'] = cfg.SSL_ENABLED if os.getenv('INVENTAR_SESSION_COOKIE_SECURE') is None else os.getenv('INVENTAR_SESSION_COOKIE_SECURE', '').strip().lower() in ('1', 'true', 'yes', 'on')
@@ -665,11 +665,14 @@ def handle_unexpected_exception(e):
def _csrf_error_response(message='CSRF token fehlt oder ist ungültig.'):
if request.is_json or request.path.startswith('/api/') or request.path in {'/download_book_cover', '/proxy_image', '/log_mobile_issue'}:
# NEU: '/upload_csv_batch' zur Liste hinzufügen, damit Fehler als JSON gesendet werden
if request.is_json or request.path.startswith('/api/') or request.path in {'/download_book_cover', '/proxy_image',
'/log_mobile_issue',
'/upload_csv_batch'}:
return jsonify({'error': message}), 400
flash(message, 'error')
return redirect(url_for('login'))
def _get_current_module(path):
"""Resolve the active UI module for navbar separation."""
mod = cfg.MODULES.get_module_for_path(path)
@@ -8903,6 +8906,7 @@ def logs():
Returns:
flask.Response: Rendered template with logs or redirect if not authenticated
"""
from modules.inventarsystem.data_protection import decrypt_text
if 'username' not in session:
flash('Ihnen ist es nicht gestattet auf dieser Internetanwendung, die eben besuchte Adrrese zu nutzen, versuchen sie es erneut nach dem sie sich mit einem berechtigten Nutzer angemeldet haben!', 'error')
return redirect(url_for('login'))
@@ -8921,9 +8925,6 @@ def logs():
# Get item details - from sample data, Item is an ID
item = it.get_item(ausleihung.get('Item'))
item_name = item.get('Name', 'Unknown Item') if item else 'Unknown Item'
# Get user details - from sample data, User is a username string
username = ausleihung.get('User', 'Unknown User')
# Determine (verified) status for display
@@ -8954,7 +8955,7 @@ def logs():
formatted_items.append({
'Item': item_name,
'User': username,
'User': decrypt_text(username),
'Start': start_date,
'End': end_date,
'Duration': duration,
@@ -8974,7 +8975,6 @@ def logs():
logs_collection = db['system_logs']
extra_logs = list(logs_collection.find({'type': {'$in': ['damage_report', 'damage_repair']}}))
from modules.inventarsystem.data_protection import decrypt_text
from bson.objectid import ObjectId
@@ -11848,3 +11848,266 @@ def test_push_notification():
except Exception as e:
app.logger.error(f'Error sending test push: {e}')
return jsonify({'success': False}), 500
@app.route('/batch_upload', methods=['GET'])
def batch_upload_page():
"""
Serves the HTML frontend for the batch CSV and image upload.
"""
# Check permissions if necessary, similar to your other routes
if 'username' not in session:
flash('Bitte melden Sie sich an.', 'error')
return redirect(url_for('login'))
return render_template('upload_batch.html')
@app.route('/upload_csv_batch', methods=['POST'])
def upload_csv_batch():
"""
Route for batch adding new items to the inventory via CSV.
Handles CSV parsing, bulk image upload with deduplication (SHA-256 hash matching),
GridFS storage, code generation, location syncing, and grouped item creation.
"""
import pandas as pd
import ast
import hashlib
username = session.get('username', 'System')
def generate_unique_batch_code(base_code, position):
"""
Generiert einen eindeutigen Code für einen Artikel innerhalb einer Serie (Batch).
:param base_code: Der Code des ersten Artikels in der Gruppe (String oder None).
:param position: Die Position des aktuellen Artikels in der Gruppe (Integer).
:return: Ein eindeutiger Code als String.
"""
if base_code:
return f"{base_code}-{position}"
else:
random_prefix = str(uuid.uuid4())[:6].upper()
return f"BATCH-{random_prefix}-{position}"
fs = get_gridfs()
upload_session_id = str(uuid.uuid4())[:8]
app.logger.info(f"Starting CSV Batch upload session {upload_session_id} - User: {username}")
# 1. Dateien aus dem Request empfangen
if 'csv_file' not in request.files:
return jsonify({"success": False, "message": "Keine CSV-Datei hochgeladen"}), 400
csv_file = request.files['csv_file']
uploaded_images = request.files.getlist('images')
# 2. CSV Einlesen und Validieren
try:
df = pd.read_csv(csv_file)
except Exception as e:
app.logger.error(f"[Upload {upload_session_id}] Fehler beim Lesen der CSV: {str(e)}")
return jsonify({"success": False, "message": f"Fehler beim Lesen der CSV: {str(e)}"}), 400
if 'Name' not in df.columns:
return jsonify({"success": False, "message": "Die CSV muss zwingend eine 'Name' Spalte enthalten."}), 400
# 3. Bilder verarbeiten & Duplikate im selben Durchlauf filtern (Hash-Matching)
image_mapping = {} # Original-Dateiname (ohne Ext) -> GridFS Filename (.webp)
processed_hashes = {} # SHA-256 Hash -> GridFS Filename (.webp)
processed_count = 0
dedup_count = 0
error_count = 0
for index, image in enumerate(uploaded_images):
if not image or not image.filename:
continue
original_secure_name = secure_filename(image.filename)
base_name_no_ext = os.path.splitext(original_secure_name)[0]
image_log_prefix = f"[Upload {upload_session_id}][Image {index + 1}/{len(uploaded_images)}]"
try:
image.seek(0)
image_bytes = image.read()
if not image_bytes:
error_count += 1
continue
# SHA-256 Hash des Bildinhalts zur Erkennung identischer Bilder
img_hash = hashlib.sha256(image_bytes).hexdigest()
if img_hash in processed_hashes:
# Bild ist identisch zu einem bereits verarbeiteten Bild im selben Batch
existing_filename = processed_hashes[img_hash]
image_mapping[base_name_no_ext] = existing_filename
dedup_count += 1
app.logger.info(f"{image_log_prefix} Duplikat erkannt ({original_secure_name}). Wiederverwendung von: {existing_filename}")
continue
# Neues Bild verarbeiten und nach WebP konvertieren
optimized_io = io.BytesIO()
with Image.open(io.BytesIO(image_bytes)) as img:
if img.mode not in ('RGB', 'RGBA'):
img = img.convert('RGBA')
max_width = 500
if img.width > max_width:
ratio = max_width / img.width
new_size = (max_width, int(img.height * ratio))
img = img.resize(new_size, Image.Resampling.LANCZOS)
img.save(optimized_io, format='WEBP', quality=85, optimize=True)
optimized_io.seek(0)
new_filename = f"{uuid.uuid4().hex}_{int(time.time())}.webp"
# In GridFS speichern
file_id = fs.put(
optimized_io,
filename=new_filename,
content_type='image/webp',
metadata={
'original_filename': original_secure_name,
'upload_session': upload_session_id,
'batch_upload': True
}
)
# In Hash-Tabelle und Mapping sichern
processed_hashes[img_hash] = new_filename
image_mapping[base_name_no_ext] = new_filename
processed_count += 1
except Exception as e:
app.logger.error(f"{image_log_prefix} Processing failed: {str(e)}")
error_count += 1
# 4. Predefined Locations laden
try:
predefined_locations = it.get_predefined_locations()
except Exception:
predefined_locations = []
# 5. Dataframe bereinigen & gruppieren
df['Name'] = df['Name'].fillna('Unbenannt').astype(str)
df = df.fillna({
'Ort': 'Unbekannt',
'Beschreibung': '',
'Code_4': '',
'Anschaffungsjahr': '',
'Anschaffungskosten': ''
})
created_item_ids = []
grouped_items = df.groupby('Name')
for name, group in grouped_items:
item_count = len(group)
series_group_id = str(uuid.uuid4()) if item_count > 1 else None
parent_item_id = None
# Basis-Code für automatisierte Seriencodes ermitteln
first_row_code = str(group.iloc[0].get('Code_4', '')).strip()
base_code = first_row_code if first_row_code else None
for position, (index, row) in enumerate(group.iterrows(), start=1):
# Ort automatisch zu predefined_locations hinzufügen, falls neu
ort_val = str(row['Ort']).strip()
if ort_val and ort_val not in predefined_locations:
try:
it.add_predefined_location(ort_val)
predefined_locations.append(ort_val)
except Exception as e:
app.logger.warning(f"Ort {ort_val} konnte nicht hinzugefügt werden: {e}")
# Bilder für diesen Artikel zuordnen
item_image_filenames = []
if 'Images' in row and pd.notna(row['Images']):
try:
img_list = ast.literal_eval(str(row['Images']))
if isinstance(img_list, list):
for img_name in img_list:
base_img_name = os.path.splitext(img_name)[0]
if base_img_name in image_mapping:
item_image_filenames.append(image_mapping[base_img_name])
else:
app.logger.warning(f"Bild {img_name} in CSV definiert, aber nicht hochgeladen.")
except (ValueError, SyntaxError):
pass
# --- NEU: BILDER-REFERENZEN PRO ARTIKEL DEDUPLIZIEREN ---
# Falls die CSV z.B. ['bild1.jpg', 'bild1.jpg'] enthält, filtern wir das hier heraus,
# damit die GridFS-Datei nicht doppelt als Referenz gespeichert wird.
unique_image_filenames = []
for img in item_image_filenames:
if img not in unique_image_filenames:
unique_image_filenames.append(img)
# --------------------------------------------------------
def parse_filter_col(col_data):
try:
res = ast.literal_eval(str(col_data))
return res if isinstance(res, list) else []
except Exception:
return []
filter_upload = parse_filter_col(row.get('Filter', '[]'))
filter_upload2 = parse_filter_col(row.get('Filter2', '[]'))
filter_upload3 = parse_filter_col(row.get('Filter3', '[]'))
reservierbar = bool(row.get('Reservierbar', False))
# Code_4 Behandlung: Falls in CSV definiert nutzen, sonst Batch-Code erzeugen
row_code = str(row.get('Code_4', '')).strip()
if row_code:
unique_code = row_code
elif 'generate_unique_batch_code' in globals():
unique_code = generate_unique_batch_code(base_code, position)
else:
unique_code = None
# DB Insert (exakt abgestimmt auf die 10 positionellen Argumente)
item_id = it.add_item(
str(row['Name']), # 1. Name
ort_val, # 2. Ort
str(row['Beschreibung']), # 3. Beschreibung
unique_image_filenames, # 4. Image Filenames (GridFS) -> HIER GEÄNDERT
filter_upload, # 5. Filter 1
filter_upload2, # 6. Filter 2
filter_upload3, # 7. Filter 3
str(row['Anschaffungsjahr']) if row['Anschaffungsjahr'] else None, # 8. Jahr
str(row['Anschaffungskosten']) if row['Anschaffungskosten'] else None, # 9. Kosten
unique_code, # 10. Unique Code / Code_4
reservierbar=reservierbar,
series_group_id=series_group_id,
series_count=item_count,
series_position=position,
is_grouped_sub_item=(position > 1),
parent_item_id=parent_item_id,
isbn=str(row.get('ISBN', '')),
item_type=str(row.get('Item_Type', 'other')),
library_category=str(row.get('Library_Category', '')),
is_library=bool(row.get('Is_Library', False))
)
if item_id:
created_item_ids.append(item_id)
if position == 1:
parent_item_id = str(item_id)
else:
app.logger.error(f"Fehler beim Erstellen von Item: {row['Name']} (Index {index})")
app.logger.info(
f"Batch Upload abgeschlossen: {len(created_item_ids)} Items erstellt. "
f"{processed_count} neue Bilder hochgeladen, {dedup_count} Bild-Duplikate zusammengeführt."
)
return jsonify({
"success": True,
"message": f"Upload erfolgreich. {len(created_item_ids)} Items importiert. {processed_count} neue Bilder gespeichert ({dedup_count} Duplikate zusammengeführt).",
"created_count": len(created_item_ids),
"images_processed": processed_count,
"images_deduplicated": dedup_count,
"images_failed": error_count
}), 200
+2 -1
View File
@@ -17,4 +17,5 @@ cryptography>=42.0.0
pywebpush
py-vapid>=1.9.0
beautifulsoup4
pywebpush
pywebpush
pandas
+1 -6
View File
@@ -4569,12 +4569,7 @@ document.addEventListener('DOMContentLoaded', ()=>{
<div class="detail-label">Code:</div>
<div class="detail-value">${escapeHtml(item.Code_4 || '-')}</div>
</div>
<div class="detail-group">
<div class="detail-label">Anzahl:</div>
<div class="detail-value">${escapeHtml(String(item.GroupedDisplayCount || 1))}</div>
</div>
${isGroupedItem ? `
<div class="detail-group">
<div class="detail-label">Verfügbar:</div>
+309
View File
@@ -0,0 +1,309 @@
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Batch Upload - CSV & Bilder</title>
<style>
:root {
--primary-color: #4a90e2;
--background-color: #f4f7f6;
--text-color: #333;
--border-radius: 8px;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background-color: var(--background-color);
color: var(--text-color);
display: flex;
justify-content: center;
align-items: center;
min-height: 100vh;
margin: 0;
padding: 20px;
box-sizing: border-box;
}
.upload-container {
background: white;
padding: 2rem;
border-radius: var(--border-radius);
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
width: 100%;
max-width: 500px;
}
h2 {
margin-top: 0;
color: var(--primary-color);
text-align: center;
}
.form-group {
margin-bottom: 1.5rem;
}
label {
display: block;
font-weight: 600;
margin-bottom: 0.5rem;
}
input[type="file"] {
display: block;
width: 100%;
padding: 0.5rem;
border: 1px dashed #ccc;
border-radius: var(--border-radius);
background: #fafafa;
cursor: pointer;
box-sizing: border-box;
}
.btn-submit {
width: 100%;
padding: 0.75rem;
background-color: var(--primary-color);
color: white;
border: none;
border-radius: var(--border-radius);
font-size: 1rem;
font-weight: 600;
cursor: pointer;
transition: background-color 0.3s ease;
}
.btn-submit:hover {
background-color: #357abd;
}
.btn-submit:disabled {
background-color: #a0c4e8;
cursor: not-allowed;
}
/* Fortschritts- und Log-Bereich */
#uploadProgress {
margin-top: 2rem;
display: none;
}
#progressText {
font-size: 1rem;
margin-bottom: 0.5rem;
color: var(--primary-color);
text-align: center;
}
progress {
width: 100%;
height: 20px;
border-radius: var(--border-radius);
}
#logList {
margin-top: 1rem;
padding: 10px;
font-size: 0.85rem;
color: #555;
max-height: 150px;
overflow-y: auto;
background: #fafafa;
border: 1px solid #ddd;
border-radius: var(--border-radius);
list-style-type: none;
}
#logList li {
margin-bottom: 5px;
padding-bottom: 5px;
border-bottom: 1px solid #eee;
}
#logList li:last-child {
border-bottom: none;
margin-bottom: 0;
padding-bottom: 0;
}
</style>
</head>
<body>
<div class="upload-container">
<h2>Inventar Batch Upload</h2>
<!-- ID auf "batchUploadForm" geändert, damit das JS es findet -->
<form id="batchUploadForm">
<div class="form-group">
<label for="csv_file">1. items.csv Datei auswählen</label>
<input type="file" id="csv_file" name="csv_file" accept=".csv" required>
</div>
<div class="form-group">
<label for="images">2. Bilder auswählen</label>
<input type="file" id="images" name="images" accept="image/*" multiple required>
<small style="color: #666; display: block; margin-top: 5px;">Du kannst mehrere Bilder markieren (Strg/Cmd gedrückt halten).</small>
</div>
<!-- ID auf "uploadBtn" geändert -->
<button type="submit" id="uploadBtn" class="btn-submit">Daten hochladen</button>
</form>
<!-- Fehlender Container für den Fortschrittsbalken und Logs hinzugefügt -->
<div id="uploadProgress">
<div id="progressText">Starte Upload...</div>
<progress id="progressBar" value="0" max="100"></progress>
<ul id="logList"></ul>
</div>
</div>
<script>
document.getElementById('batchUploadForm').addEventListener('submit', async function(e) {
e.preventDefault();
const csvInput = document.getElementById('csv_file');
const imageInput = document.getElementById('images');
const uploadBtn = document.getElementById('uploadBtn');
const progressContainer = document.getElementById('uploadProgress');
const progressBar = document.getElementById('progressBar');
const progressText = document.getElementById('progressText');
const logList = document.getElementById('logList');
if (!csvInput.files.length) {
alert("Bitte wähle eine CSV-Datei aus.");
return;
}
uploadBtn.disabled = true;
progressContainer.style.display = 'block';
logList.innerHTML = '';
const log = (msg) => {
const li = document.createElement('li');
li.textContent = msg;
logList.appendChild(li);
logList.scrollTop = logList.scrollHeight; // Auto-scroll
};
const csvFile = csvInput.files[0];
const allImages = Array.from(imageInput.files);
const BATCH_SIZE = 50;
try {
// 1. CSV-Datei lesen
const csvText = await csvFile.text();
// 2. CSV in Zeilen aufteilen
let rows = csvText.split(/\r?\n/).filter(row => row.trim().length > 0);
if (rows.length <= 1) {
throw new Error("CSV-Datei ist leer oder enthält nur Kopfzeilen.");
}
const header = rows[0];
let dataRows = rows.slice(1);
// 3. Client-seitige Deduplizierung (Entfernt exakte Duplikat-Zeilen)
const uniqueRowsSet = new Set();
const uniqueDataRows = [];
let duplicateCount = 0;
for (const row of dataRows) {
if (uniqueRowsSet.has(row)) {
duplicateCount++;
} else {
uniqueRowsSet.add(row);
uniqueDataRows.push(row);
}
}
log(`${uniqueDataRows.length} einzigartige Einträge gefunden. ${duplicateCount} Duplikate entfernt.`);
// Den Index der "Images" Spalte finden
const headers = header.split(',');
const imagesColIndex = headers.findIndex(h => h.trim().replace(/['"]/g, '') === 'Images');
// 4. In Batches (Häppchen) aufteilen
const batches = [];
for (let i = 0; i < uniqueDataRows.length; i += BATCH_SIZE) {
batches.push(uniqueDataRows.slice(i, i + BATCH_SIZE));
}
progressBar.max = batches.length;
progressBar.value = 0;
// 5. Batches nacheinander hochladen
for (let b = 0; b < batches.length; b++) {
const batchRows = batches[b];
progressText.textContent = `Lade Batch ${b + 1} von ${batches.length} hoch...`;
log(`Bereite Batch ${b + 1} vor (${batchRows.length} Artikel)...`);
// CSV für diesen Batch neu zusammensetzen
const batchCsvText = [header, ...batchRows].join('\n');
const batchCsvBlob = new Blob([batchCsvText], { type: 'text/csv' });
// Benötigte Bilder für diesen Batch extrahieren
const requiredImageNames = new Set();
if (imagesColIndex !== -1) {
batchRows.forEach(row => {
const cols = row.split(',');
if (cols[imagesColIndex]) {
try {
let imgStr = cols[imagesColIndex].trim().replace(/^"|"$/g, '').replace(/'/g, '"');
if (imgStr.startsWith('[') && imgStr.endsWith(']')) {
const parsedImages = JSON.parse(imgStr);
parsedImages.forEach(img => requiredImageNames.add(img));
}
} catch (err) {
console.warn("Konnte Bild-Array nicht parsen in Zeile:", row);
}
}
});
}
// Bilder auf die für diesen Batch benötigten filtern
const batchImages = allImages.filter(img => requiredImageNames.has(img.name));
// FormData zusammenbauen
const formData = new FormData();
formData.append('csv_file', batchCsvBlob, `batch_${b+1}.csv`);
batchImages.forEach(img => {
formData.append('images', img);
});
// An Server senden
try {
const response = await fetch('/upload_csv_batch', {
method: 'POST',
body: formData
});
const result = await response.json();
if (!response.ok || !result.success) {
throw new Error(result.message || `Server antwortete mit Status ${response.status}`);
}
log(`Batch ${b + 1} erfolgreich: ${result.message}`);
} catch (batchErr) {
log(`Fehler in Batch ${b + 1}: ${batchErr.message}`);
alert(`Upload wurde bei Batch ${b + 1} aufgrund eines Fehlers abgebrochen. Prüfe die Logs.`);
break; // Stoppt weitere Uploads, wenn einer fehlschlägt
}
progressBar.value = b + 1;
}
progressText.textContent = "Upload-Vorgang abgeschlossen!";
uploadBtn.disabled = false;
} catch (error) {
alert("Fehler bei der Verarbeitung des Uploads: " + error.message);
log("Fehler: " + error.message);
uploadBtn.disabled = false;
}
});
</script>
</body>
</html>
+2 -1
View File
@@ -17,4 +17,5 @@ cryptography>=42.0.0
pywebpush
py-vapid>=1.9.0
beautifulsoup4
pywebpush
pywebpush
pandas