fix hltb worker

backlogger/hltb.py

@@ -36,13 +36,15 @@ def fetch(game_name):
 
 
 def apply_to_item(item):
-    """Fetch HLTB data and save it onto item. Silently does nothing on failure."""
+    """Fetch HLTB data and save it onto item. Always marks hltb_fetched=True."""
     if item.category != 'games' or not item.name:
         return
     data = fetch(item.name)
-    if data is None:
+    fields = ['hltb_fetched']
-        return
+    item.hltb_fetched = True
+    if data is not None:
         item.hltb_main = data['main']
         item.hltb_extra = data['extra']
         item.hltb_complete = data['complete']
-    item.save(update_fields=['hltb_main', 'hltb_extra', 'hltb_complete'])
+        fields += ['hltb_main', 'hltb_extra', 'hltb_complete']
+    item.save(update_fields=fields)

backlogger/management/commands/run_hltb_worker.py

@@ -0,0 +1,42 @@
+import time
+import logging
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from datetime import timedelta
+from backlogger.models import Item
+from backlogger import hltb as hltb_api
+
+logger = logging.getLogger(__name__)
+
+INITIAL_DELAY_SECONDS = 20
+BETWEEN_LOOKUPS_SECONDS = 5
+IDLE_POLL_SECONDS = 10
+
+
+class Command(BaseCommand):
+    help = 'Background worker: fetches HLTB data for newly created game items.'
+
+    def handle(self, *args, **options):
+        self.stdout.write('HLTB worker started.')
+        while True:
+            cutoff = timezone.now() - timedelta(seconds=INITIAL_DELAY_SECONDS)
+            item = (
+                Item.objects
+                .filter(category=Item.GAMES, hltb_fetched=False, created_at__lte=cutoff)
+                .order_by('created_at')
+                .first()
+            )
+            if item is None:
+                time.sleep(IDLE_POLL_SECONDS)
+                continue
+
+            self.stdout.write(f'Fetching HLTB for: {item.name} (id={item.pk})')
+            try:
+                hltb_api.apply_to_item(item)
+            except Exception as e:
+                logger.exception('HLTB lookup failed for item %s', item.pk)
+                # Mark as fetched anyway to avoid retrying indefinitely
+                item.hltb_fetched = True
+                item.save(update_fields=['hltb_fetched'])
+
+            time.sleep(BETWEEN_LOOKUPS_SECONDS)

backlogger/migrations/0011_item_hltb_fetched.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('backlogger', '0010_item_steam_appid'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='item',
+            name='hltb_fetched',
+            field=models.BooleanField(default=False),
+        ),
+    ]

backlogger/models.py

@@ -67,6 +67,7 @@ class Item(models.Model):
     hltb_main = models.FloatField(null=True, blank=True)
     hltb_extra = models.FloatField(null=True, blank=True)
     hltb_complete = models.FloatField(null=True, blank=True)
+    hltb_fetched = models.BooleanField(default=False)
 
     class Meta:
         ordering = ['-favorite', 'name']

backlogger/views.py

@@ -176,7 +176,7 @@ def steam_import(request):
             continue
         hours = game['hours']
         progress = min(100.0, hours) if hours > 0 else 0.0
-        item = Item.objects.create(
+        Item.objects.create(
             user=request.user,
             category=Item.GAMES,
             name=game['name'],
@@ -184,7 +184,6 @@ def steam_import(request):
             progress_percent=progress,
             steam_appid=game['appid'],
         )
-        hltb_api.apply_to_item(item)
         imported += 1
 
     del request.session['steam_games']

entrypoint.sh

@@ -13,6 +13,8 @@ else:
     print(f'{Mineral.objects.count()} minerals already loaded')
 "
 
+python manage.py run_hltb_worker &
+
 exec gunicorn kboris.wsgi:application \
     --bind 0.0.0.0:8080 \
     --workers 2 \