elfeed-tube-utils.el (20307B)
1 ;;; elfeed-tube-utils.el --- utilities for elfeed-tube -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2022 Karthik Chikmagalur 4 5 ;; Author: Karthik Chikmagalur <karthikchikmagalur@gmail.com> 6 ;; Keywords: multimedia, convenience 7 8 ;; SPDX-License-Identifier: UNLICENSE 9 10 ;; This file is NOT part of GNU Emacs. 11 12 ;;; Commentary: 13 ;; 14 ;; Utilities for Elfeed Tube. 15 ;; 16 ;;; Code: 17 (require 'rx) 18 (require 'aio) 19 (require 'elfeed) 20 21 (declare-function elfeed-tube--with-label "elfeed-tube") 22 (declare-function elfeed-tube--fetch-1 "elfeed-tube") 23 (declare-function elfeed-tube-show "elfeed-tube") 24 (declare-function elfeed-tube-curl-enqueue "elfeed-tube") 25 (declare-function elfeed-tube--attempt-log "elfeed-tube") 26 (declare-function elfeed-tube-log "elfeed-tube") 27 (declare-function elfeed-tube--get-invidious-url "elfeed-tube") 28 (declare-function elfeed-tube--nrotate-invidious-servers "elfeed-tube") 29 30 (defvar elfeed-tube-youtube-regexp) 31 (defvar elfeed-tube--api-videos-path) 32 (defvar elfeed-tube--max-retries) 33 34 (defsubst elfeed-tube--ensure-list (var) 35 "Ensure VAR is a list." 36 (if (listp var) var (list var))) 37 38 (cl-defstruct (elfeed-tube-channel (:constructor elfeed-tube-channel-create) 39 (:copier nil)) 40 "Struct to hold youtube channel information." 41 query author url feed) 42 43 ;;;###autoload (autoload 'elfeed-tube-add-feeds "elfeed-tube-utils" "Add youtube feeds to the Elfeed database by QUERIES." t nil) 44 (aio-defun elfeed-tube-add-feeds (queries &optional _) 45 "Add youtube feeds to the Elfeed database by QUERIES. 46 47 Each query can be a video, playlist or channel URL and the 48 corresponding channel feed will be selected. It can also be a 49 search term and the best match will be found. You will be asked 50 to finalize the results before committing them to Elfeed. 51 52 When called interactively, multiple queries can be provided by 53 separating them with the `crm-separator', typically 54 comma (\",\"). Search terms cannot include the `crm-separator'. 55 56 When called noninteractively, it accepts a query or a list of 57 queries." 58 (interactive 59 (list (completing-read-multiple 60 "Video, Channel, Playlist URLs or search queries: " 61 #'ignore) 62 current-prefix-arg)) 63 (message "Finding RSS feeds, hold tight!") 64 (let ((channels (aio-await (elfeed-tube-add--get-channels queries)))) 65 (elfeed-tube-add--display-channels channels))) 66 67 (defsubst elfeed-tube--video-p (cand) 68 "Check if CAND is a Youtube video URL." 69 (string-match 70 (concat 71 elfeed-tube-youtube-regexp 72 (rx (zero-or-one "watch?v=") 73 (group (1+ (not "&"))))) 74 cand)) 75 76 (defsubst elfeed-tube--playlist-p (cand) 77 "Check if CAND is a Youtube playlist URL." 78 (string-match 79 (concat 80 elfeed-tube-youtube-regexp 81 "playlist\\?list=" 82 (rx (group (1+ (not "&"))))) 83 cand)) 84 85 (defsubst elfeed-tube--channel-p (cand) 86 "Check if CAND is a Youtube channel URL." 87 (string-match 88 (concat 89 elfeed-tube-youtube-regexp 90 (rx "channel/" 91 (group (1+ (not "&"))))) 92 cand)) 93 94 (aio-defun elfeed-tube-add--get-channels (queries) 95 (let* ((fetches (aio-make-select)) 96 (queries (elfeed-tube--ensure-list queries)) 97 (playlist-base-url 98 "https://www.youtube.com/feeds/videos.xml?playlist_id=") 99 (channel-base-url 100 "https://www.youtube.com/feeds/videos.xml?channel_id=") 101 channels) 102 103 ;; Add all promises to fetches, an aio-select 104 (dolist (q queries channels) 105 (setq q (string-trim q)) 106 (cond 107 ((elfeed-tube--channel-p q) 108 (let* ((chan-id (match-string 1 q)) 109 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 110 "/api/v1/channels/" 111 chan-id 112 "?fields=author,authorUrl")) 113 (feed (concat channel-base-url chan-id))) 114 (aio-select-add fetches 115 (elfeed-tube--with-label 116 `(:type channel :feed ,feed :query ,q) 117 #'elfeed-tube--aio-fetch 118 api-url #'elfeed-tube--nrotate-invidious-servers)))) 119 120 ((string-match 121 (concat elfeed-tube-youtube-regexp "c/" "\\([^?&]+\\)") q) 122 ;; Interpret channel url as search query 123 (let* ((search-url "/api/v1/search") 124 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 125 search-url 126 "?q=" (url-hexify-string (match-string 1 q)) 127 "&type=channel&page=1"))) 128 (aio-select-add fetches 129 (elfeed-tube--with-label 130 `(:type search :query ,q) 131 #'elfeed-tube--aio-fetch 132 api-url #'elfeed-tube--nrotate-invidious-servers)))) 133 134 ((elfeed-tube--playlist-p q) 135 (let* ((playlist-id (match-string 1 q)) 136 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 137 "/api/v1/playlists/" 138 playlist-id 139 "?fields=title,author")) 140 (feed (concat playlist-base-url playlist-id))) 141 (aio-select-add fetches 142 (elfeed-tube--with-label 143 `(:type playlist :feed ,feed :query ,q) 144 #'elfeed-tube--aio-fetch 145 api-url #'elfeed-tube--nrotate-invidious-servers)))) 146 147 ((elfeed-tube--video-p q) 148 (if-let* ((video-id (match-string 1 q)) 149 (videos-url "/api/v1/videos/") 150 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 151 videos-url 152 video-id 153 "?fields=author,authorUrl,authorId"))) 154 (aio-select-add fetches 155 (elfeed-tube--with-label 156 `(:type video :query ,q) 157 #'elfeed-tube--aio-fetch 158 api-url #'elfeed-tube--nrotate-invidious-servers)) 159 (push (elfeed-tube-channel-create :query q) 160 channels))) 161 162 (t ;interpret as search query 163 (let* ((search-url "/api/v1/search") 164 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 165 search-url 166 "?q=" (url-hexify-string q) 167 "&type=channel&page=1"))) 168 (aio-select-add fetches 169 (elfeed-tube--with-label 170 `(:type search :query ,q) 171 #'elfeed-tube--aio-fetch 172 api-url #'elfeed-tube--nrotate-invidious-servers)))))) 173 174 ;; Resolve all promises in the aio-select 175 (while (aio-select-promises fetches) 176 (pcase-let* ((`(,label . ,data) 177 (aio-await (aio-await (aio-select fetches)))) 178 (q (plist-get label :query)) 179 (feed (plist-get label :feed))) 180 (pcase (plist-get label :type) 181 ('channel 182 (if-let ((author (plist-get data :author)) 183 (author-url (plist-get data :authorUrl))) 184 (push (elfeed-tube-channel-create 185 :query q :author author 186 :url q 187 :feed feed) 188 channels) 189 (push (elfeed-tube-channel-create :query q :feed feed) 190 channels))) 191 192 ('playlist 193 (if-let ((title (plist-get data :title)) 194 (author (plist-get data :author))) 195 (push (elfeed-tube-channel-create 196 :query q :author title :url q 197 :feed feed) 198 channels) 199 (push (elfeed-tube-channel-create 200 :query q :url q 201 :feed feed) 202 channels))) 203 ('video 204 (if-let* ((author (plist-get data :author)) 205 (author-id (plist-get data :authorId)) 206 (author-url (plist-get data :authorUrl)) 207 (feed (concat channel-base-url author-id))) 208 (push (elfeed-tube-channel-create 209 :query q :author author 210 :url (concat "https://www.youtube.com" author-url) 211 :feed feed) 212 channels) 213 (push (elfeed-tube-channel-create :query (plist-get label :query)) 214 channels))) 215 ('search 216 (if-let* ((chan-1 (and (> (length data) 0) 217 (aref data 0))) 218 (author (plist-get chan-1 :author)) 219 (author-id (plist-get chan-1 :authorId)) 220 (author-url (plist-get chan-1 :authorUrl)) 221 (feed (concat channel-base-url author-id))) 222 (push (elfeed-tube-channel-create 223 :query q :author author 224 :url (concat "https://www.youtube.com" author-url) 225 :feed feed) 226 channels) 227 (push (elfeed-tube-channel-create :query q) 228 channels)))))) 229 230 (nreverse channels))) 231 232 (defun elfeed-tube-add--display-channels (channels) 233 "Summarize found Youtube channel feeds CHANNELS." 234 (let ((buffer (get-buffer-create "*Elfeed-Tube Channels*")) 235 (notfound (propertize "Not found!" 'face 'error))) 236 (with-current-buffer buffer 237 (let ((inhibit-read-only t)) (erase-buffer)) 238 (elfeed-tube-channels-mode) 239 (setq 240 tabulated-list-entries 241 (cl-loop for channel in channels 242 for n upfrom 1 243 for author = (if-let ((url (elfeed-tube-channel-url channel))) 244 (list (elfeed-tube-channel-author channel) 245 'mouse-face 'highlight 246 'action 247 #'elfeed-tube-add--visit-channel 248 'follow-link t 249 'help-echo (elfeed-tube-channel-url channel)) 250 notfound) 251 for feed = (or (elfeed-tube-channel-feed channel) notfound) 252 collect 253 `(,n 254 [,author 255 ,(replace-regexp-in-string 256 elfeed-tube-youtube-regexp "" 257 (elfeed-tube-channel-query channel)) 258 ,feed]))) 259 (tabulated-list-init-header) 260 (tabulated-list-print) 261 (goto-address-mode 1) 262 263 (goto-char (point-max)) 264 265 (let ((inhibit-read-only t) 266 (fails (cl-reduce 267 (lambda (sum ch) 268 (+ sum 269 (or (and (elfeed-tube-channel-feed ch) 0) 1))) 270 channels :initial-value 0)) 271 (continue (propertize "C-c C-c" 'face 'help-key-binding)) 272 (continue-extra (propertize "C-u C-c C-c" 'face 'help-key-binding)) 273 (cancel-q (propertize "q" 'face 'help-key-binding)) 274 (cancel (propertize "C-c C-k" 'face 'help-key-binding)) 275 (copy (propertize "C-c C-w" 'face 'help-key-binding))) 276 277 (let ((inhibit-message t)) 278 (toggle-truncate-lines 1)) 279 (insert "\n") 280 (when (> fails 0) 281 (insert (propertize 282 (format "%d queries could not be resolved.\n\n" fails) 283 'face 'error) 284 " " continue ": Add found feeds to the Elfeed database, ignoring the failures.\n" 285 " " continue-extra ": Add found feeds, fetch entries from them and open Elfeed.\n")) 286 (when (= fails 0) 287 (insert 288 (propertize 289 "All queries resolved successfully.\n\n" 290 'face 'success) 291 " " continue ": Add all feeds to the Elfeed database.\n" 292 " " continue-extra ": Add all feeds, fetch entries from them and open Elfeed.\n" 293 " " copy ": Copy the list of feed URLs as a list\n")) 294 (insert "\n" cancel-q " or " cancel ": Quit and cancel this operation.")) 295 296 (goto-char (point-min)) 297 298 (funcall 299 (if (bound-and-true-p demo-mode) 300 #'switch-to-buffer 301 #'display-buffer) 302 buffer)))) 303 304 (defun elfeed-tube-add--visit-channel (button) 305 "Activate BUTTON." 306 (browse-url (button-get button 'help-echo))) 307 308 ;; (elfeed-tube-add--display-channels my-channels) 309 310 (defun elfeed-tube-add--confirm (&optional arg) 311 "Confirm the addition of visible Youtube feeds to the Elfeed database. 312 313 With optional prefix argument ARG, update these feeds and open Elfeed 314 afterwards." 315 (interactive "P") 316 (cl-assert (derived-mode-p 'elfeed-tube-channels-mode)) 317 (let* ((channels tabulated-list-entries)) 318 (let ((inhibit-message t)) 319 (cl-loop for channel in channels 320 for (_ _ feed) = (append (cadr channel) nil) 321 do (elfeed-add-feed feed :save t))) 322 (message "Added to elfeed-feeds.") 323 (when arg (elfeed)))) 324 325 (define-derived-mode elfeed-tube-channels-mode tabulated-list-mode 326 "Elfeed Tube Channels" 327 (setq tabulated-list-use-header-line t ; default to no header 328 header-line-format nil 329 ;; tabulated-list--header-string nil 330 tabulated-list-format 331 '[("Channel" 22 t) 332 ("Query" 32 t) 333 ("Feed URL" 30 nil)])) 334 335 (defvar elfeed-tube-channels-mode-map 336 (let ((map (make-sparse-keymap))) 337 (define-key map (kbd "C-c C-k") #'kill-buffer) 338 (define-key map (kbd "C-c C-c") #'elfeed-tube-add--confirm) 339 (define-key map (kbd "C-c C-w") #'elfeed-tube-add--copy) 340 map)) 341 342 (defun elfeed-tube-add--copy () 343 "Copy visible Youtube feeds to the kill ring as a list. 344 345 With optional prefix argument ARG, update these feeds and open Elfeed 346 afterwards." 347 (interactive) 348 (cl-assert (derived-mode-p 'elfeed-tube-channels-mode)) 349 (let* ((channels tabulated-list-entries)) 350 (cl-loop for channel in channels 351 for (_ _ feed) = (append (cadr channel) nil) 352 collect feed into feeds 353 finally (kill-new (prin1-to-string feeds))) 354 (message "Feed URLs saved to kill-ring."))) 355 356 (aio-defun elfeed-tube--aio-fetch (url &optional next desc attempts) 357 "Fetch URL asynchronously using `elfeed-curl-retrieve'. 358 359 If successful (HTTP 200), return the JSON-parsed result as a 360 plist. 361 362 Otherwise, call the function NEXT (with no arguments) and try 363 ATTEMPTS more times. Return nil if all attempts fail. DESC is a 364 description string to print to the elfeed-tube log allong with 365 any other error messages. 366 367 This function returns a promise." 368 (let ((attempts (or attempts (1+ elfeed-tube--max-retries)))) 369 (when (> attempts 0) 370 (let* ((response 371 (aio-await (elfeed-tube-curl-enqueue url :method "GET"))) 372 (content (plist-get response :content)) 373 (status (plist-get response :status-code)) 374 (error-msg (plist-get response :error-message))) 375 (cond 376 ((= status 200) 377 (condition-case nil 378 (json-parse-string content :object-type 'plist) 379 ((json-parse-error error) 380 (elfeed-tube-log 'error "[Search] JSON malformed (%s)" 381 (elfeed-tube--attempt-log attempts)) 382 (and (functionp next) (funcall next)) 383 (aio-await 384 (elfeed-tube--aio-fetch url next desc (1- attempts)))))) 385 (t (elfeed-tube-log 'error "[Search][%s]: %s (%s)" error-msg url 386 (elfeed-tube--attempt-log attempts)) 387 (and (functionp next) (funcall next)) 388 (aio-await 389 (elfeed-tube--aio-fetch url next desc (1- attempts))))))))) 390 391 (aio-defun elfeed-tube--fake-entry (url &optional force-fetch) 392 (string-match (concat elfeed-tube-youtube-regexp 393 (rx (zero-or-one "watch?v=") 394 (group (1+ (not (or "&" "?")))))) 395 url) 396 (if-let ((video-id (match-string 1 url))) 397 (progn 398 (message "Creating a video summary...") 399 (cl-letf* ((elfeed-show-unique-buffers t) 400 (elfeed-show-entry-switch #'display-buffer) 401 (elfeed-tube-save-indicator nil) 402 (elfeed-tube-auto-save-p nil) 403 (api-data (aio-await 404 (elfeed-tube--aio-fetch 405 (concat (aio-await (elfeed-tube--get-invidious-url)) 406 elfeed-tube--api-videos-path 407 video-id 408 "?fields=" 409 ;; "videoThumbnails,descriptionHtml,lengthSeconds," 410 "title,author,authorUrl,published") 411 #'elfeed-tube--nrotate-invidious-servers))) 412 (feed-id (concat "https://www.youtube.com/feeds/videos.xml?channel_id=" 413 (nth 1 (split-string (plist-get api-data :authorUrl) 414 "/" t)))) 415 (author `((:name ,(plist-get api-data :author) 416 :uri ,feed-id))) 417 (entry 418 (elfeed-entry--create 419 :link url 420 :title (plist-get api-data :title) 421 :id `("www.youtube.com" . ,(concat "yt:video:" video-id)) 422 :date (plist-get api-data :published) 423 :tags '(youtube) 424 :content-type 'html 425 :meta `(:authors ,author) 426 :feed-id feed-id)) 427 ((symbol-function 'elfeed-entry-feed) 428 (lambda (_) 429 (elfeed-feed--create 430 :id feed-id 431 :url feed-id 432 :title (plist-get api-data :author) 433 :author author)))) 434 (aio-await (elfeed-tube--fetch-1 entry force-fetch)) 435 (with-selected-window (elfeed-show-entry entry) 436 (message "Summary created for video: \"%s\"" 437 (elfeed-entry-title entry)) 438 (setq-local elfeed-show-refresh-function 439 (lambda () (interactive) 440 (elfeed-tube-show)) 441 elfeed-tube-save-indicator nil)))) 442 (message "Not a youtube video URL, aborting."))) 443 444 (defsubst elfeed-tube--line-at-point () 445 "Get line around point." 446 (buffer-substring (line-beginning-position) (line-end-position))) 447 448 (defun elfeed-tube-next-heading (&optional arg) 449 "Jump to the next heading in an Elfeed entry. 450 451 With numeric prefix argument ARG, jump forward that many times. 452 If ARG is negative, jump backwards instead." 453 (interactive "p") 454 (unless arg (setq arg 1)) 455 (catch 'return 456 (dotimes (_ (abs arg)) 457 (when (> arg 0) (end-of-line)) 458 (if-let ((match 459 (funcall (if (> arg 0) 460 #'text-property-search-forward 461 #'text-property-search-backward) 462 'face `(shr-h1 shr-h2 shr-h3 463 message-header-name elfeed-tube-chapter-face) 464 (lambda (tags face) 465 (cl-loop for x in (if (consp face) face (list face)) 466 thereis (memq x tags))) 467 t))) 468 (goto-char 469 (if (> arg 0) (prop-match-beginning match) (prop-match-end match))) 470 (throw 'return nil)) 471 (when (< arg 0) (beginning-of-line))) 472 (beginning-of-line) 473 (point))) 474 475 (defun elfeed-tube-prev-heading (&optional arg) 476 "Jump to the previous heading in an Elfeed entry. 477 478 With numeric prefix argument ARG, jump backward that many times. 479 If ARG is negative, jump forward instead." 480 (interactive "p") 481 (elfeed-tube-next-heading (- (or arg 1)))) 482 483 (provide 'elfeed-tube-utils) 484 ;;; elfeed-tube-utils.el ends here