Upload New File

Artwork-Fetching.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Artwork Fetching Tool
+
+Options:
+ 1. Export from test-release.cdb to JSON
+ 2. Dump IDs from JSON to CSV (merging existing Official mappings)
+ 3. Fetch artwork images for IDs (with multi-art prompting)
+
+Configuration paths at top — no prompts for file locations except for multi-art choices.
+"""
+import os
+import sys
+import json
+import sqlite3
+import csv
+import requests
+from tqdm import tqdm
+
+# ===== Configuration =====
+SCRIPT_DIR       = os.path.dirname(os.path.realpath(__file__))
+DB_PATH          = os.path.join(SCRIPT_DIR, 'test-release.cdb')
+TRANSFORMED_JSON = os.path.join(SCRIPT_DIR, 'cards_transformed.json')
+IDS_CSV          = os.path.join(SCRIPT_DIR, 'IDs.csv')
+MANIFEST_JSON    = os.path.join(SCRIPT_DIR, 'manifest.json')
+OUTPUT_DIR       = os.path.join(SCRIPT_DIR, 'pics')
+DEBUG            = False  # set True for verbose logging
+
+def export_db():
+    """Export cards from .cdb into JSON with fields id, name, desc."""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('SELECT id, name, desc FROM texts')
+    rows = cursor.fetchall()
+    cards = [{'id': cid, 'name': name, 'desc': desc} for cid, name, desc in rows]
+    conn.close()
+    with open(TRANSFORMED_JSON, 'w', encoding='utf-8') as f:
+        json.dump(cards, f, ensure_ascii=False, indent=2)
+    print(f'Exported {len(cards)} cards to {TRANSFORMED_JSON}')
+
+def dump_ids():
+    """Dump card IDs from transformed JSON to CSV, preserving existing Official mappings."""
+    existing = {}
+    if os.path.exists(IDS_CSV):
+        with open(IDS_CSV, newline='', encoding='utf-8-sig') as f:
+            reader = csv.DictReader(f)
+            if 'Pre-Release' in reader.fieldnames and 'Official' in reader.fieldnames:
+                for row in reader:
+                    pre = row['Pre-Release'].strip()
+                    off = row['Official'].strip()
+                    if pre:
+                        existing[pre] = off
+    with open(TRANSFORMED_JSON, 'r', encoding='utf-8') as f:
+        cards = json.load(f)
+
+    os.makedirs(os.path.dirname(IDS_CSV) or '.', exist_ok=True)
+    with open(IDS_CSV, 'w', newline='', encoding='utf-8') as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(['Name','Pre-Release','Official'])
+        for c in cards:
+            name = c.get('name','')
+            pre  = str(c.get('id',''))
+            off  = existing.get(pre, '')
+            writer.writerow([name, pre, off])
+    print(f'Dumped {len(cards)} IDs to {IDS_CSV} (preserved {len(existing)} mappings)')
+
+def load_ids_list():
+    ids = []
+    with open(IDS_CSV, newline='', encoding='utf-8-sig') as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            pre = row.get('Pre-Release','').strip()
+            off = row.get('Official','').strip()
+            # prefer Official if present
+            ids.append(off or pre)
+    return ids
+
+def load_manifest():
+    with open(MANIFEST_JSON, encoding='utf-8') as f:
+        data = json.load(f)
+    # unwrap nested if needed
+    if not any(k.isdigit() for k in data.keys()):
+        for v in data.values():
+            if isinstance(v, dict) and any(k.isdigit() for k in v.keys()):
+                return v
+    return data
+
+def get_artwork_url(entry):
+    # pick in order bestOCG, bestArt, bestTCG
+    for key in ('bestOCG','bestArt','bestTCG'):
+        url = entry.get(key)
+        if url:
+            return url
+    return None
+
+def normalize_url(url):
+    if url.startswith('//'):
+        return 'https:' + url
+    if url.startswith('/'):
+        return 'https://' + url.lstrip('/')
+    return url if url.startswith(('http://','https://')) else 'https://' + url
+
+def fetch_artworks():
+    """
+    Download artworks, prompting when multiple versions exist.
+    """
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+    all_ids  = load_ids_list()
+    manifest = load_manifest()
+
+    # Build list of IDs present in manifest
+    valid_ids = []
+    for id_str in all_ids:
+        if id_str in manifest:
+            valid_ids.append(id_str)
+        else:
+            try:
+                n = str(int(id_str))
+                if n in manifest:
+                    valid_ids.append(n)
+                    if DEBUG:
+                        print(f"Normalized {id_str} → {n}")
+                    continue
+            except ValueError:
+                pass
+            if DEBUG:
+                print(f"Skipping missing ID {id_str}")
+
+    # Process each ID
+    for id_str in tqdm(valid_ids, desc='Fetching artworks', total=len(valid_ids)):
+        entry = manifest.get(id_str)
+        # find numeric versions
+        versions = sorted([v for v in entry.keys() if v.isdigit()], key=int)
+        count = len(versions)
+
+        # decide which versions to download
+        if count <= 1:
+            to_dl = ['1']
+        else:
+            default_n = count if count <= 3 else 3
+            resp = input(f"ID {id_str} has {count} artworks. Download last {default_n}? [Y/n]: ").strip().lower()
+            if resp not in ('', 'y', 'yes'):
+                if DEBUG:
+                    print(f"Skipped extra arts for {id_str}")
+                continue
+            # pick last default_n versions
+            to_dl = [str(v) for v in versions[-default_n:]]
+
+        # download selected
+        for ver in to_dl:
+            art_entry = entry.get(ver, {})
+            url_raw = get_artwork_url(art_entry)
+            if not url_raw:
+                if DEBUG:
+                    print(f"No URL for {id_str} v{ver}")
+                continue
+            url = normalize_url(url_raw)
+            fname = f"{id_str}_{ver}.png" if count > 1 else f"{id_str}.png"
+            outp  = os.path.join(OUTPUT_DIR, fname)
+            if os.path.exists(outp):
+                continue
+            try:
+                r = requests.get(url, stream=True, timeout=10)
+                r.raise_for_status()
+                with open(outp, 'wb') as f:
+                    for chunk in r.iter_content(8192):
+                        f.write(chunk)
+            except Exception as e:
+                if DEBUG:
+                    print(f"Error downloading {url}: {e}")
+
+    print(f"Fetched artwork for {len(valid_ids)} cards to {OUTPUT_DIR}")
+
+def main():
+    print("Select an option:")
+    print("1: Export from test-release.cdb")
+    print("2: Dump IDs from JSON")
+    print("3: Fetch Artwork")
+    choice = input("Select Option: ").strip()
+    if choice == '1':
+        export_db()
+    elif choice == '2':
+        dump_ids()
+    elif choice == '3':
+        fetch_artworks()
+    else:
+        print("Invalid choice.")
+        sys.exit(1)
+
+if __name__=='__main__':
+    main()