YouTube: Support V3 API via user-supplied key (bug 1999)

This adds auto-discovery of the channel ID and new-style feed for old-style
(username-based) feed URLs when the V3 API key is available, and also adds an
extra menu item for migrating subscriptions.

Author: thp

share/gpodder/ui/gtk/gpodder.ui

@@ -233,6 +233,13 @@
           </object>
           <accelerator key="S" modifiers="GDK_CONTROL_MASK"/>
         </child>
+        <child>
+          <object class="GtkAction" id="item_update_youtube_subscriptions">
+            <property name="name">item_update_youtube_subscriptions</property>
+            <property name="label" translatable="yes">Update YouTube subscriptions</property>
+            <signal handler="on_update_youtube_subscriptions_activate" name="activate"/>
+          </object>
+        </child>
         <child>
           <object class="GtkAction" id="menuView">
             <property name="name">menuView</property>
@@ -367,6 +374,7 @@
         </menu>
         <menu action="menuExtras">
           <menuitem action="item_sync"/>
+          <menuitem action="item_update_youtube_subscriptions"/>
         </menu>
         <menu action="menuView">
           <menuitem action="itemShowToolbar"/>

share/gpodder/ui/gtk/gpodderpreferences.ui

@@ -52,7 +52,7 @@
                   <object class="GtkTable" id="table_players">
                     <property name="column_spacing">6</property>
                     <property name="n_columns">3</property>
-                    <property name="n_rows">4</property>
+                    <property name="n_rows">5</property>
                     <property name="row_spacing">6</property>
                     <property name="visible">True</property>
                     <child>
@@ -162,15 +162,62 @@
                       </packing>
                     </child>
                     <child>
-                      <object class="GtkLabel" id="label_preferred_vimeo_format">
+                      <object class="GtkLabel" id="label_youtube_api_key">
                         <property name="visible">True</property>
                         <property name="can_focus">False</property>
                         <property name="xalign">0</property>
-                        <property name="label" translatable="yes">Preferred Vimeo format:</property>
+                        <property name="label" translatable="yes">YouTube API key (v3):</property>
+                      </object>
+                      <packing>
+                        <property name="left_attach">0</property>
+                        <property name="right_attach">1</property>
+                        <property name="top_attach">3</property>
+                        <property name="bottom_attach">4</property>
+                        <property name="x_options">fill</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkEntry" id="entry_youtube_api_key">
+                        <property name="visible">True</property>
+                        <signal handler="on_youtube_api_key_changed" name="changed"/>
+                      </object>
+                      <packing>
+                        <property name="left_attach">1</property>
+                        <property name="right_attach">2</property>
+                        <property name="top_attach">3</property>
+                        <property name="bottom_attach">4</property>
+                        <property name="x_options">fill</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkButton" id="button_youtube_api_key">
+                        <property name="visible">True</property>
+                        <signal name="clicked" handler="on_button_youtube_api_key_clicked"/>
+                        <child>
+                          <object class="GtkImage" id="image_youtube_api_key">
+                            <property name="stock">gtk-jump-to</property>
+                            <property name="visible">True</property>
+                          </object>
+                        </child>
                       </object>
                       <packing>
                         <property name="top_attach">3</property>
                         <property name="bottom_attach">4</property>
+                        <property name="left_attach">2</property>
+                        <property name="right_attach">3</property>
+                        <property name="x_options">fill</property>
+                      </packing>
+                    </child>
+                    <child>
+                      <object class="GtkLabel" id="label_preferred_vimeo_format">
+                        <property name="visible">True</property>
+                        <property name="can_focus">False</property>
+                        <property name="xalign">0</property>
+                        <property name="label" translatable="yes">Preferred Vimeo format:</property>
+                      </object>
+                      <packing>
+                        <property name="top_attach">4</property>
+                        <property name="bottom_attach">5</property>
                         <property name="x_options">fill</property>
                       </packing>
                     </child>
@@ -183,8 +230,8 @@
                       <packing>
                         <property name="left_attach">1</property>
                         <property name="right_attach">2</property>
-                        <property name="top_attach">3</property>
-                        <property name="bottom_attach">4</property>
+                        <property name="top_attach">4</property>
+                        <property name="bottom_attach">5</property>
                       </packing>
                     </child>
                   </object>

src/gpodder/config.py

@@ -191,6 +191,7 @@
     'youtube': {
         'preferred_fmt_id': 18, # default fmt_id (see fallbacks in youtube.py)
         'preferred_fmt_ids': [], # for advanced uses (custom fallback sequence)
+        'api_key_v3': '', # API key, register for one at https://developers.google.com/youtube/v3/
     },
 
     'vimeo': {

src/gpodder/gtkui/desktop/preferences.py

@@ -314,6 +314,7 @@ def new(self):
         self.entry_username.set_text(self._config.mygpo.username)
         self.entry_password.set_text(self._config.mygpo.password)
         self.entry_caption.set_text(self._config.mygpo.device.caption)
+        self.entry_youtube_api_key.set_text(self._config.youtube.api_key_v3)
 
         # Disable mygpo sync while the dialog is open
         self._config.mygpo.enabled = False
@@ -596,6 +597,12 @@ def on_enabled_toggled(self, widget):
         # Only update indirectly (see on_dialog_destroy)
         self._enable_mygpo = widget.get_active()
 
+    def on_youtube_api_key_changed(self, widget):
+        self._config.youtube.api_key_v3 = widget.get_text()
+
+    def on_button_youtube_api_key_clicked(self, widget):
+        util.open_website('https://developers.google.com/youtube/v3/')
+
     def on_username_changed(self, widget):
         self._config.mygpo.username = widget.get_text()
 

src/gpodder/gtkui/main.py

@@ -2177,6 +2177,17 @@ def add_podcast_list(self, podcasts, auth_tokens=None):
         queued, failed, existing, worked, authreq = [], [], [], [], []
         for input_title, input_url in podcasts:
             url = util.normalize_feed_url(input_url)
+
+            # Check if it's a YouTube feed, and if we have an API key, auto-resolve the channel
+            if url is not None and self.config.youtube.api_key_v3:
+                xurl, xuser = youtube.for_each_feed_pattern(lambda url, channel: (url, channel), url, (None, None))
+                if xurl is not None and xuser is not None:
+                    logger.info('Getting channels for YouTube user %s (%s)', xuser, xurl)
+                    new_urls = youtube.get_channels_for_user(xuser, self.config.youtube.api_key_v3)
+                    logger.debug('YouTube channels retrieved: %r', new_urls)
+                    if len(new_urls) == 1:
+                        url = new_urls[0]
+
             if url is None:
                 # Fail this one because the URL is not valid
                 failed.append(input_url)
@@ -3484,6 +3495,52 @@ def on_sync_to_device_activate(self, widget, episodes=None, force_played=True):
 
         self.sync_ui.on_synchronize_episodes(self.channels, episodes, force_played)
 
+    def on_update_youtube_subscriptions_activate(self, widget):
+        if not self.config.youtube.api_key_v3:
+            if self.show_confirmation('\n'.join((_('Please register a YouTube API key and set it in the preferences.'),
+                                                 _('Would you like to set up an API key now?'))), _('API key required')):
+                self.on_itemPreferences_activate(self, widget)
+            return
+
+        failed_urls = []
+        migrated_users = []
+        for podcast in self.channels:
+            url, user = youtube.for_each_feed_pattern(lambda url, channel: (url, channel), podcast.url, (None, None))
+            if url is not None and user is not None:
+                try:
+                    logger.info('Getting channels for YouTube user %s (%s)', user, url)
+                    new_urls = youtube.get_channels_for_user(user, self.config.youtube.api_key_v3)
+                    logger.debug('YouTube channels retrieved: %r', new_urls)
+
+                    if len(new_urls) != 1:
+                        failed_urls.append(url, _('No unique URL found'))
+                        continue
+
+                    new_url = new_urls[0]
+                    if new_url in set(x.url for x in self.model.get_podcasts()):
+                        failed_urls.append((url, _('Already subscribed')))
+                        continue
+
+                    logger.info('New feed location: %s => %s', url, new_url)
+                    podcast.url = new_url
+                    podcast.save()
+                    migrated_users.append(user)
+                except Exception as e:
+                    logger.error('Exception happened while updating download list.', exc_info=True)
+                    self.show_message(_('Make sure the API key is correct. Error: %(message)s') % {'message': str(e)},
+                                      _('Error getting YouTube channels'), important=True)
+
+        if migrated_users:
+            self.show_message('\n'.join(migrated_users), _('Successfully migrated subscriptions'))
+        elif not failed_urls:
+            self.show_message(_('Subscriptions are up to date'))
+
+        if failed_urls:
+            self.show_message('\n'.join([_('These URLs failed:'), ''] + ['{}: {}'.format(url, message)
+                                                                         for url, message in failed_urls]),
+                              _('Could not migrate some subscriptions'), important=True)
+
+
 def main(options=None):
     gobject.threads_init()
     gobject.set_application_name('gPodder')

src/gpodder/youtube.py

@@ -78,6 +78,10 @@
 ]
 formats_dict = dict(formats)
 
+V3_API_ENDPOINT = 'https://www.googleapis.com/youtube/v3'
+CHANNEL_VIDEOS_XML = 'https://www.youtube.com/feeds/videos.xml'
+
+
 class YouTubeError(Exception): pass
 
 
@@ -188,6 +192,7 @@ def for_each_feed_pattern(func, url, fallback_result):
         'http[s]?://(?:[a-z]+\.)?youtube\.com/channel/([_a-z0-9]+)',
         'http[s]?://(?:[a-z]+\.)?youtube\.com/rss/user/([a-z0-9]+)/videos\.rss',
         'http[s]?://gdata.youtube.com/feeds/users/([^/]+)/uploads',
+        'http[s]?://(?:[a-z]+\.)?youtube\.com/feeds/videos.xml?channel_id=([a-z0-9]+)',
     ]
 
     for pattern in CHANNEL_MATCH_PATTERNS:
@@ -219,3 +224,8 @@ def return_user_cover(url, channel):
         return None
 
     return for_each_feed_pattern(return_user_cover, url, None)
+
+def get_channels_for_user(username, api_key_v3):
+    stream = util.urlopen('{}/channels?forUsername={}&part=id&key={}'.format(V3_API_ENDPOINT, username, api_key_v3))
+    data = json.load(stream)
+    return ['{}?channel_id={}'.format(CHANNEL_VIDEOS_XML, item['id']) for item in data['items']]