elfeed-tube-fill.el (15853B)
1 ;;; elfeed-tube-fill.el --- Back-fill elfeed-tube feeds -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2022 Karthik Chikmagalur 4 5 ;; Author: Karthik Chikmagalur <karthikchikmagalur@gmail.com> 6 ;; Keywords: multimedia, convenience 7 8 ;; SPDX-License-Identifier: UNLICENSE 9 10 ;; This file is NOT part of GNU Emacs. 11 12 ;;; Commentary: 13 ;; 14 ;; This file contains commands to back-fill Elfeed YouTube feeds. Back-filling a 15 ;; feed fetches all historical entries for the corresponding YouTube channel or 16 ;; playlist and adds them to the Elfeed database. Youtube RSS feeds generally 17 ;; contain only the latest 15 entries. 18 ;; 19 ;; Call `elfeed-tube-fill-feeds' in an Elfeed search or entry buffer to 20 ;; back-fill entries for the corresponding feed. You can select a region of 21 ;; entries to fill all the corresponding feeds. 22 ;; 23 ;;; Code: 24 25 (require 'elfeed-tube) 26 27 (declare-function elfeed-tube--get-entries "elfeed-tube") 28 (defvar elfeed-tube--api-channels-videos-path "/api/v1/channels/videos/%s") 29 (defvar elfeed-tube--api-playlists-videos-path "/api/v1/playlists/%s") 30 (defvar elfeed-tube--fill-tags nil 31 "Alist of Elfeed feed-ids and tags to add. 32 33 These tags (list of symbols) will be added when back-filling the 34 corresponding feed.") 35 36 37 (cl-deftype elfeed-tube--fill-api-data () 38 `(satisfies 39 (lambda (coll) 40 (and (vectorp coll) 41 (or (= coll 0) 42 (cl-every (lambda (vd) (and (plist-get vd :videoId) 43 (plist-get vd :published) 44 (plist-get vd :title))) 45 coll)))))) 46 47 ;;;###autoload (autoload 'elfeed-tube-fill-feeds "elfeed-tube-utils" "Fetch and add all channel videos for ENTRIES' feeds." t nil) 48 (aio-defun elfeed-tube-fill-feeds (entries &optional interactive-p) 49 "Fetch and add all channel videos for ENTRIES' feeds. 50 51 YouTube RSS feeds generally contain only the latest 15 entries. 52 Use this command to fetch and add to Elfeed all videos 53 corresponding a channel or playlist. 54 55 ENTRIES is the entry at point or visited entry, or the list of 56 selected entries if the region is active. 57 58 When called interactively, INTERACTIVE-P is t and a summary 59 window will be shown before taking any action." 60 (interactive (list (elfeed-tube--ensure-list (elfeed-tube--get-entries)) 61 t)) 62 (let ((feeds (cl-reduce 63 (lambda (accum entry) 64 (if-let* ((feed (elfeed-entry-feed entry)) 65 ((memq feed accum))) 66 accum 67 (cons feed accum))) 68 entries 69 :initial-value nil))) 70 (if interactive-p 71 (elfeed-tube--fill-display-feeds feeds) 72 (aio-await (elfeed-tube--fill-feeds feeds))))) 73 74 (aio-defun elfeed-tube--fill-feeds (feeds) 75 "Find videos corresponding to the channels/playlists for Elfeed feeds FEEDS. 76 77 Videos not already present will be added to the Elfeed database." 78 (cl-check-type feeds (and (not null) (not atom))) 79 (cl-check-type (car feeds) elfeed-feed) 80 81 (dolist (feed feeds) 82 (elfeed-tube-log 'debug "[(fill-feeds): Backfilling feed: %s]" (elfeed-feed-title feed)) 83 (let ((elfeed-tube-auto-fetch-p nil) 84 (feed-url (elfeed-feed-url feed)) 85 (feed-id (elfeed-feed-id feed)) 86 (feed-title (elfeed-feed-title feed)) 87 (add-count) 88 (feed-entries-to-add 89 (thread-first 90 (elfeed-tube--fill-feed feed) 91 (aio-await) 92 (cl-delete-duplicates :key (lambda (x) (plist-get x :videoId)) :test #'string=) 93 (vconcat) 94 (elfeed-tube--fill-feed-dates) 95 (aio-await)))) 96 97 (cl-check-type feed-entries-to-add elfeed-tube--fill-api-data 98 "Missing video attributes (ID, Title or Publish Date).") 99 100 (if (= (length feed-entries-to-add) 0) 101 (message "Nothing to retrieve for feed \"%s\" (%s)" feed-title feed-url) 102 (setq add-count (length feed-entries-to-add)) 103 104 (condition-case error 105 (thread-last 106 feed-entries-to-add 107 (cl-map 'list (apply-partially #'elfeed-tube--entry-create feed-id)) 108 (cl-map 'list (lambda (entry) 109 (setf (elfeed-entry-tags entry) 110 (or (alist-get feed-id elfeed-tube--fill-tags 111 nil nil #'equal) 112 '(unread))) 113 entry)) 114 (elfeed-db-add)) 115 (error (elfeed-handle-parse-error feed-url error))) 116 ;; (prin1 feed-entries-to-add (get-buffer "*scratch*")) 117 (elfeed-tube-log 'debug "[(elfeed-db): Backfilling feed: %s][Added %d videos]" 118 feed-title add-count) 119 (message "Retrieved %d missing videos for feed \"%s\" (%s)" 120 add-count feed-title feed-url) 121 (run-hook-with-args 'elfeed-update-hooks feed-url))))) 122 123 ;; feed: elfeed-feed struct, page: int or nil -> vector(plist entries for feed videos not in db) 124 (aio-defun elfeed-tube--fill-feed (feed &optional page) 125 "Find videos corresponding to the channel/playlist for Elfeed feed FEED. 126 127 Return video metadata as a vector of plists. Metadata 128 corresponding to videos already in the Elfeed database are 129 filtered out. 130 131 PAGE corresponds to the page number of results requested from the API." 132 (cl-check-type feed elfeed-feed "An Elfeed Feed") 133 (cl-check-type page (or null (integer 0 *)) "A positive integer.") 134 135 (if-let* ((page (or page 1)) 136 (feed-url (elfeed-feed-url feed)) 137 (feed-title (elfeed-feed-title feed)) 138 (api-path (cond ((string-match "playlist_id=\\(.*?\\)/*$" feed-url) 139 (concat 140 (format elfeed-tube--api-playlists-videos-path 141 (match-string 1 feed-url)) 142 "?fields=videos(title,videoId,author)" 143 "&page=" (number-to-string (or page 1)))) 144 ((string-match "channel_id=\\(.*?\\)/*$" feed-url) 145 (concat 146 (format elfeed-tube--api-channels-videos-path 147 (match-string 1 feed-url)) 148 "?fields=" 149 "title,videoId,author,published" 150 "&sort_by=newest" 151 "&page=" (number-to-string (or page 1)))) 152 (t (elfeed-tube-log 'error "[Malformed/Not YouTube feed: %s][%s]" 153 feed-title feed-url) 154 nil))) 155 (feed-type (cond ((string-match "playlist_id=\\(.*?\\)/*$" feed-url) 'playlist) 156 ((string-match "channel_id=\\(.*?\\)/*$" feed-url) 'channel)))) 157 (let ((feed-entry-video-ids 158 (mapcar (lambda (e) (elfeed-tube--url-video-id (elfeed-entry-link e))) 159 (elfeed-feed-entries feed))) 160 (feed-id (elfeed-feed-id feed))) 161 (if-let* 162 ((api-data 163 (aio-await 164 (elfeed-tube--aio-fetch 165 (concat (aio-await (elfeed-tube--get-invidious-url)) api-path) 166 #'elfeed-tube--nrotate-invidious-servers))) 167 (api-data (pcase feed-type 168 ('channel api-data) 169 ('playlist 170 (cl-check-type api-data (and (not null) list)) 171 (plist-get api-data :videos)))) 172 ((> (length api-data) 0))) 173 (progn 174 (cl-check-type api-data elfeed-tube--fill-api-data) 175 (elfeed-tube-log 'debug "[Backfilling: page %d][Fetched: %d entries]" 176 (or page 1) (length api-data)) 177 (vconcat 178 (cl-delete-if ;remove entries already in db 179 (lambda (elt) (member (plist-get elt :videoId) feed-entry-video-ids)) 180 api-data) 181 (aio-await (elfeed-tube--fill-feed feed (1+ page))))) 182 (make-vector 0 0))) 183 (elfeed-tube-log 'error "[Malformed/Not Youtube feed: %s][%s]" feed-title feed-url))) 184 185 ;; api-data: vector(plist entries for feed videos) -> vector(plist entries for 186 ;; feed videos with correct dates.) 187 (aio-defun elfeed-tube--fill-feed-dates (api-data) 188 "Add or correct dates for videos in API-DATA. 189 190 API-DATA is a vector of plists, one per video. This function 191 returns a vector of plists with video publish dates 192 corrected/added as the value of the plist's :published key." 193 (cl-check-type api-data elfeed-tube--fill-api-data) 194 (let ((date-queries) 195 (feed-videos-map (make-hash-table :test 'equal)) 196 (fix-count 0)) 197 198 (if (= (length api-data) 0) 199 api-data 200 (progn 201 (elfeed-tube-log 'debug "[Fixing publish dates]") 202 (cl-loop for video-plist across api-data 203 for video-id = (plist-get video-plist :videoId) 204 do (puthash video-id video-plist feed-videos-map) 205 do (push (elfeed-tube--with-label 206 video-id #'elfeed-tube--aio-fetch 207 (concat (aio-wait-for (elfeed-tube--get-invidious-url)) 208 elfeed-tube--api-videos-path 209 video-id "?fields=published")) 210 date-queries)) 211 212 (dolist (promise (nreverse date-queries)) 213 (pcase-let* ((`(,video-id . ,corrected-date) (aio-await promise)) 214 (video-plist (gethash video-id feed-videos-map))) 215 216 (plist-put video-plist :published (plist-get corrected-date :published)) 217 (cl-incf fix-count))) 218 219 (elfeed-tube-log 'debug "[Fixed publish dates for %d videos]" fix-count) 220 221 (vconcat (hash-table-values feed-videos-map)))))) 222 223 ;; Back-fill GUI 224 225 (defsubst elfeed-tube--fill-tags-strings (taglist) 226 "Convert a list of tags TAGLIST to a comma separated string." 227 (mapconcat 228 (lambda (s) (propertize (symbol-name s) 229 'face 'elfeed-search-tag-face)) 230 taglist ",")) 231 232 (defun elfeed-tube--fill-display-feeds (feeds) 233 "Produce a summary of Elfeed FEEDS to be back-filled. 234 235 Back-filling a YouTube feed will fetch all its videos not 236 presently available in its RSS feed or in the Elfeed database." 237 (let ((buffer (get-buffer-create "*Elfeed-Tube Channels*"))) 238 (with-current-buffer buffer 239 (let ((inhibit-read-only t)) (erase-buffer)) 240 (elfeed-tube-channels-mode) 241 242 (setq tabulated-list-use-header-line t ; default to no header 243 header-line-format nil 244 ;; tabulated-list--header-string nil 245 tabulated-list-format 246 '[("Channel" 22 t) 247 ("#Entries" 10 t) 248 ("Tags to apply" 30 nil) 249 ("Feed URL" 30 nil)]) 250 251 (setq 252 tabulated-list-entries 253 (cl-loop for feed in feeds 254 for n upfrom 1 255 for feed-url = (elfeed-feed-url feed) 256 for channel-id = (progn (string-match "=\\(.*?\\)$" feed-url) 257 (match-string 1 feed-url)) 258 for feed-title = (list (propertize (elfeed-feed-title feed) 259 'feed feed) 260 'mouse-face 'highlight 261 'action 262 #'elfeed-tube-add--visit-channel 263 'follow-link t 264 'help-echo 265 (or (and channel-id 266 (concat 267 "https://www.youtube.com/channel/" 268 channel-id)) 269 "")) 270 for feed-count = (number-to-string (length (elfeed-feed-entries feed))) 271 for feed-tags = (if-let ((taglist 272 (alist-get (elfeed-feed-id feed) 273 elfeed-tube--fill-tags nil t #'equal))) 274 (elfeed-tube--fill-tags-strings taglist) 275 (propertize "unread" 'face 'elfeed-search-tag-face)) 276 collect 277 `(,n 278 [,feed-title 279 ,feed-count 280 ,feed-tags 281 ,feed-url]))) 282 283 (tabulated-list-init-header) 284 (tabulated-list-print) 285 (goto-address-mode 1) 286 287 (goto-char (point-max)) 288 (let ((inhibit-read-only t) 289 (continue (propertize "C-c C-c" 'face 'help-key-binding)) 290 (cancel-q (propertize "q" 'face 'help-key-binding)) 291 (cancel (propertize "C-c C-k" 'face 'help-key-binding))) 292 293 (let ((inhibit-message t)) (toggle-truncate-lines 1)) 294 (insert "\n") 295 (insert 296 " " (propertize "t" 'face 'help-key-binding) 297 " or " (propertize "+" 'face 'help-key-binding) 298 ": Set tags to apply to back-filled entries for feed.\n\n" 299 " " continue ": Add All (historical) videos from these channels to Elfeed.\n" 300 cancel-q " or " cancel ": Quit and cancel this operation.\n")) 301 302 (goto-char (point-min)) 303 304 (use-local-map (copy-keymap elfeed-tube-channels-mode-map)) 305 (local-set-key (kbd "C-c C-c") #'elfeed-tube--fill-confirm) 306 (local-set-key (kbd "+") #'elfeed-tube--fill-tags-add) 307 (local-set-key (kbd "t") #'elfeed-tube--fill-tags-add) 308 309 (display-buffer 310 buffer `(nil 311 (window-height . ,#'fit-window-to-buffer) 312 (body-function . ,#'select-window)))))) 313 314 (defun elfeed-tube--fill-tags-add () 315 "Add tags to back-filled entries fetched for feed at point." 316 (interactive) 317 (when-let* ((entry (tabulated-list-get-entry)) 318 (feed (thread-last (aref entry 0) 319 (car) 320 (get-text-property 0 'feed))) 321 (title (elfeed-feed-title feed)) 322 (id (elfeed-feed-id feed)) 323 (tags (read-from-minibuffer 324 (format "Add tags for \"%s\" (comma separated): " title) 325 (thread-last 326 (or (alist-get id elfeed-tube--fill-tags nil t #'equal) '(unread)) 327 (mapcar #'symbol-name) 328 (funcall (lambda (tg) (string-join tg ",")))))) 329 (taglist (thread-last (split-string tags "," t "[ \f\t\n\r\v]+") 330 (mapcar #'intern-soft) 331 (elfeed-normalize-tags)))) 332 (setf (alist-get (elfeed-feed-id feed) elfeed-tube--fill-tags nil nil #'equal) 333 taglist) 334 (tabulated-list-set-col 2 (elfeed-tube--fill-tags-strings taglist)))) 335 336 (aio-defun elfeed-tube--fill-confirm () 337 "Back-fill video entries for the displayed Elfeed feeds." 338 (interactive) 339 (cl-assert (derived-mode-p 'elfeed-tube-channels-mode)) 340 (cl-loop for table-entry in tabulated-list-entries 341 for feed-title = (car (aref (cadr table-entry) 0)) 342 collect (get-text-property 0 'feed feed-title) into feeds 343 finally do (elfeed-tube-log 'debug "[(fill-confirm-feeds): %S]" 344 (mapcar #'elfeed-feed-title feeds)) 345 finally do 346 (progn 347 (quit-window 'kill-buffer) 348 (message "Backfilling YouTube feeds...") 349 (aio-await (elfeed-tube--fill-feeds feeds)) 350 (message "Backfilling Youtube feeds... done.")))) 351 352 (provide 'elfeed-tube-fill) 353 ;;; elfeed-tube-fill.el ends here