Move update routine to a subcommand of manage.py

odyseescraper/management/commands/odyseescrape.py

@@ -0,0 +1,46 @@
+#! /usr/bin/env python3
+import uuid
+import odyseescraper
+from django.core.management.base import BaseCommand, CommandError
+from django.db import models
+
+class Command(BaseCommand):
+    help = "Updates all OdyseeChannel objects, creating OdyseeRelease objects for anything new"
+
+    def handle(self, *args, **options):
+        self.stdout.write('Updating Channels...')
+        updates = 0
+        failures = []
+        for channel in odyseescraper.models.OdyseeChannel.objects.all():
+            print(f'{str(channel)}...')
+            self.stdout.write('Getting releases...')
+            releases = odyseescraper.odysee.odysee_get_releases(channel.handle)
+            for release in releases:
+                try:
+                    data = releases[release]
+                    if odyseescraper.models.OdyseeRelease.objects.filter(pk=release).exists():
+                        self.stdout.write(f'Skipping existing release: {str(channel)} - {data['title']}')
+                        continue
+                    a = odyseescraper.models.OdyseeRelease.objects.get_or_create(
+                        id = release,
+                        channel = channel.id,
+                        name = data['title'],
+                        description = data['description'],
+                        url = data['url'],
+                        thumbnail = data['thumbnail'],
+                    )
+                    self.stdout.write(f'Imported release: {str(channel)} - {data['title']}')
+                    updates += 1
+                except Exception as e:
+                    self.stdout.write(self.style.WARNING(f'Failed to update {str(channel)}: {e}'))
+                    failures.append({"channel": str(channel),"item":releases[release]["title"],"error":e})
+        if failures:
+            self.stdout.write(self.style.ERROR('Errors occurred while importing data:'))
+            for error in failures:
+                self.stdout.write(self.style.ERROR(f'{error["channel"]} - {error["item"]} - {error["error"]}'))
+        else:
+            if updates > 0:
+                self.stdout.write(self.style.SUCCESS(f'Imported {updates} new releases'))
+            else:
+                self.stdout.write(self.style.SUCCESS('No new imports'))
+        pass

odyseescraper/models.py

@@ -18,26 +18,6 @@ class OdyseeChannel(models.Model):
     def __str__(self):
         return self.name
 
-    def save(self, *args, **kwargs):
-        # First, save the field
-        super().save(*args, **kwargs)
-        # Then we're going to do a super kludge to try and add its releases
-        try:
-            releases = odysee.odysee_get_releases(self.handle)
-            for release in releases:
-                data = releases[release]
-                a = OdyseeRelease.objects.get_or_create(
-                    id = release,
-                    channel = self,
-                    name = data['title'],
-                    description = data['description'],
-                    url = data['url'],
-                    thumbnail = data['thumbnail']
-                )
-        except Exception as e:
-            print(f'Failed to get releases for this channel: {e}')
-        return
-
 class OdyseeRelease(models.Model):
     id = models.CharField(primary_key=True, max_length=512, default=uuid.uuid4, editable=False, db_index=True)
     channel = models.ForeignKey(OdyseeChannel, on_delete=models.CASCADE, db_index=True)

odyseescraper/odysee.py

@@ -12,7 +12,6 @@ def odysee_get_releases(handle):
     releases = {}
     try:
         for i in range(1,20):
-            print(f'Examining page {i} for handle {handle}')
             payload = {
                 "method": "claim_search",
                 "params": {
@@ -35,7 +34,6 @@ def odysee_get_releases(handle):
                     pass
                 else:
                     print(f'Unknown value type, continuing: item["value_type"]')
-                print(f'Importing item {item["claim_id"]}')
                 releases[item["claim_id"]] = {
                     "name": item.get("name", "Unnamed Release"),
                     "title": item["value"].get("title", "Untitled Release"),