elfeed-tube-utils.el (20927B)
1 ;;; elfeed-tube-utils.el --- utilities for elfeed-tube -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2022 Karthik Chikmagalur 4 5 ;; Author: Karthik Chikmagalur <karthikchikmagalur@gmail.com> 6 ;; Keywords: multimedia, convenience 7 8 ;; SPDX-License-Identifier: UNLICENSE 9 10 ;; This file is NOT part of GNU Emacs. 11 12 ;;; Commentary: 13 ;; 14 ;; Utilities for Elfeed Tube. 15 ;; 16 ;;; Code: 17 (require 'rx) 18 (require 'aio) 19 (require 'elfeed) 20 21 (declare-function elfeed-tube--with-label "elfeed-tube") 22 (declare-function elfeed-tube--fetch-1 "elfeed-tube") 23 (declare-function elfeed-tube-show "elfeed-tube") 24 (declare-function elfeed-tube-curl-enqueue "elfeed-tube") 25 (declare-function elfeed-tube--attempt-log "elfeed-tube") 26 (declare-function elfeed-tube-log "elfeed-tube") 27 (declare-function elfeed-tube--get-invidious-url "elfeed-tube") 28 (declare-function elfeed-tube--nrotate-invidious-servers "elfeed-tube") 29 30 (defvar elfeed-tube-youtube-regexp) 31 (defvar elfeed-tube--api-videos-path) 32 (defvar elfeed-tube--max-retries) 33 34 (defsubst elfeed-tube--ensure-list (var) 35 "Ensure VAR is a list." 36 (if (listp var) var (list var))) 37 38 (cl-defstruct (elfeed-tube-channel (:constructor elfeed-tube-channel-create) 39 (:copier nil)) 40 "Struct to hold youtube channel information." 41 query author url feed) 42 43 ;;;###autoload (autoload 'elfeed-tube-add-feeds "elfeed-tube-utils" "Add youtube feeds to the Elfeed database by QUERIES." t nil) 44 (aio-defun elfeed-tube-add-feeds (queries &optional _) 45 "Add youtube feeds to the Elfeed database by QUERIES. 46 47 Each query can be a video, playlist or channel URL and the 48 corresponding channel feed will be selected. It can also be a 49 search term and the best match will be found. You will be asked 50 to finalize the results before committing them to Elfeed. 51 52 When called interactively, multiple queries can be provided by 53 separating them with the `crm-separator', typically 54 comma (\",\"). Search terms cannot include the `crm-separator'. 55 56 When called noninteractively, it accepts a query or a list of 57 queries." 58 (interactive 59 (list (completing-read-multiple 60 "Video, Channel, Playlist URLs or search queries: " 61 #'ignore) 62 current-prefix-arg)) 63 (message "Finding RSS feeds, hold tight!") 64 (let ((channels (aio-await (elfeed-tube-add--get-channels queries)))) 65 (elfeed-tube-add--display-channels channels))) 66 67 (defsubst elfeed-tube--video-p (cand) 68 "Check if CAND is a Youtube video URL." 69 (string-match 70 (concat 71 elfeed-tube-youtube-regexp 72 (rx (zero-or-one "watch?v=") 73 (group (1+ (not "&"))))) 74 cand)) 75 76 (defsubst elfeed-tube--playlist-p (cand) 77 "Check if CAND is a Youtube playlist URL." 78 (string-match 79 (concat 80 elfeed-tube-youtube-regexp 81 "playlist\\?list=" 82 (rx (group (1+ (not "&"))))) 83 cand)) 84 85 (defsubst elfeed-tube--channel-p (cand) 86 "Check if CAND is a Youtube channel URL." 87 (string-match 88 (concat 89 elfeed-tube-youtube-regexp 90 (rx "channel/" 91 (group (1+ (not "&"))))) 92 cand)) 93 94 (aio-defun elfeed-tube-add--get-channels (queries) 95 (let* ((fetches (aio-make-select)) 96 (queries (elfeed-tube--ensure-list queries)) 97 (playlist-base-url 98 "https://www.youtube.com/feeds/videos.xml?playlist_id=") 99 (channel-base-url 100 "https://www.youtube.com/feeds/videos.xml?channel_id=") 101 channels) 102 103 ;; Add all promises to fetches, an aio-select 104 (dolist (q queries channels) 105 (setq q (string-trim q)) 106 (cond 107 ((elfeed-tube--channel-p q) 108 (let* ((chan-id (match-string 1 q)) 109 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 110 "/api/v1/channels/" 111 chan-id 112 "?fields=author,authorUrl")) 113 (feed (concat channel-base-url chan-id))) 114 (aio-select-add fetches 115 (elfeed-tube--with-label 116 `(:type channel :feed ,feed :query ,q) 117 #'elfeed-tube--aio-fetch 118 api-url #'elfeed-tube--nrotate-invidious-servers)))) 119 120 ((string-match 121 (concat elfeed-tube-youtube-regexp "c/" "\\([^?&]+\\)") q) 122 ;; Interpret channel url as search query 123 (let* ((search-url "/api/v1/search") 124 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 125 search-url 126 "?q=" (url-hexify-string (match-string 1 q)) 127 "&type=channel&page=1"))) 128 (aio-select-add fetches 129 (elfeed-tube--with-label 130 `(:type search :query ,q) 131 #'elfeed-tube--aio-fetch 132 api-url #'elfeed-tube--nrotate-invidious-servers)))) 133 134 ((elfeed-tube--playlist-p q) 135 (let* ((playlist-id (match-string 1 q)) 136 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 137 "/api/v1/playlists/" 138 playlist-id 139 "?fields=title,author")) 140 (feed (concat playlist-base-url playlist-id))) 141 (aio-select-add fetches 142 (elfeed-tube--with-label 143 `(:type playlist :feed ,feed :query ,q) 144 #'elfeed-tube--aio-fetch 145 api-url #'elfeed-tube--nrotate-invidious-servers)))) 146 147 ((elfeed-tube--video-p q) 148 (if-let* ((video-id (match-string 1 q)) 149 (videos-url "/api/v1/videos/") 150 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 151 videos-url 152 video-id 153 "?fields=author,authorUrl,authorId"))) 154 (aio-select-add fetches 155 (elfeed-tube--with-label 156 `(:type video :query ,q) 157 #'elfeed-tube--aio-fetch 158 api-url #'elfeed-tube--nrotate-invidious-servers)) 159 (push (elfeed-tube-channel-create :query q) 160 channels))) 161 162 (t ;interpret as search query 163 (let* ((search-url "/api/v1/search") 164 (api-url (concat (aio-await (elfeed-tube--get-invidious-url)) 165 search-url 166 "?q=" (url-hexify-string q) 167 "&type=channel&page=1"))) 168 (aio-select-add fetches 169 (elfeed-tube--with-label 170 `(:type search :query ,q) 171 #'elfeed-tube--aio-fetch 172 api-url #'elfeed-tube--nrotate-invidious-servers)))))) 173 174 ;; Resolve all promises in the aio-select 175 (while (aio-select-promises fetches) 176 (pcase-let* ((`(,label . ,data) 177 (aio-await (aio-await (aio-select fetches)))) 178 (q (plist-get label :query)) 179 (feed (plist-get label :feed))) 180 (pcase (plist-get label :type) 181 ('channel 182 (if-let ((author (plist-get data :author)) 183 (author-url (plist-get data :authorUrl))) 184 (push (elfeed-tube-channel-create 185 :query q :author author 186 :url q 187 :feed feed) 188 channels) 189 (push (elfeed-tube-channel-create :query q :feed feed) 190 channels))) 191 192 ('playlist 193 (if-let ((title (plist-get data :title)) 194 (author (plist-get data :author))) 195 (push (elfeed-tube-channel-create 196 :query q :author title :url q 197 :feed feed) 198 channels) 199 (push (elfeed-tube-channel-create 200 :query q :url q 201 :feed feed) 202 channels))) 203 ('video 204 (if-let* ((author (plist-get data :author)) 205 (author-id (plist-get data :authorId)) 206 (author-url (plist-get data :authorUrl)) 207 (feed (concat channel-base-url author-id))) 208 (push (elfeed-tube-channel-create 209 :query q :author author 210 :url (concat "https://www.youtube.com" author-url) 211 :feed feed) 212 channels) 213 (push (elfeed-tube-channel-create :query (plist-get label :query)) 214 channels))) 215 ('search 216 (if-let* ((chan-1 (and (> (length data) 0) 217 (aref data 0))) 218 (author (plist-get chan-1 :author)) 219 (author-id (plist-get chan-1 :authorId)) 220 (author-url (plist-get chan-1 :authorUrl)) 221 (feed (concat channel-base-url author-id))) 222 (push (elfeed-tube-channel-create 223 :query q :author author 224 :url (concat "https://www.youtube.com" author-url) 225 :feed feed) 226 channels) 227 (push (elfeed-tube-channel-create :query q) 228 channels)))))) 229 230 (nreverse channels))) 231 232 (defun elfeed-tube-add--display-channels (channels) 233 "Summarize found Youtube channel feeds CHANNELS." 234 (let ((buffer (get-buffer-create "*Elfeed-Tube Channels*")) 235 (notfound (propertize "Not found!" 'face 'error))) 236 (with-current-buffer buffer 237 (let ((inhibit-read-only t)) (erase-buffer)) 238 (elfeed-tube-channels-mode) 239 (setq 240 tabulated-list-entries 241 (cl-loop for channel in channels 242 for n upfrom 1 243 for author = (if-let ((url (elfeed-tube-channel-url channel))) 244 (list (elfeed-tube-channel-author channel) 245 'mouse-face 'highlight 246 'action 247 #'elfeed-tube-add--visit-channel 248 'follow-link t 249 'help-echo (elfeed-tube-channel-url channel)) 250 notfound) 251 for feed = (or (elfeed-tube-channel-feed channel) notfound) 252 collect 253 `(,n 254 [,author 255 ,(replace-regexp-in-string 256 elfeed-tube-youtube-regexp "" 257 (elfeed-tube-channel-query channel)) 258 ,feed]))) 259 (setq tabulated-list-format 260 '[("Channel" 22 t) 261 ("Query" 32 t) 262 ("Feed URL" 30 nil)]) 263 264 (tabulated-list-init-header) 265 (tabulated-list-print) 266 (goto-address-mode 1) 267 268 (goto-char (point-max)) 269 270 (let ((inhibit-read-only t) 271 (fails (cl-reduce 272 (lambda (sum ch) 273 (+ sum 274 (or (and (elfeed-tube-channel-feed ch) 0) 1))) 275 channels :initial-value 0)) 276 (continue (propertize "C-c C-c" 'face 'help-key-binding)) 277 (continue-extra (propertize "C-u C-c C-c" 'face 'help-key-binding)) 278 (cancel-q (propertize "q" 'face 'help-key-binding)) 279 (cancel (propertize "C-c C-k" 'face 'help-key-binding)) 280 (copy (propertize "C-c C-w" 'face 'help-key-binding))) 281 282 (let ((inhibit-message t)) 283 (toggle-truncate-lines 1)) 284 (insert "\n") 285 (when (> fails 0) 286 (insert (propertize 287 (format "%d queries could not be resolved.\n\n" fails) 288 'face 'error) 289 " " continue ": Add found feeds to the Elfeed database, ignoring the failures.\n" 290 " " continue-extra ": Add found feeds, fetch entries from them and open Elfeed.\n")) 291 (when (= fails 0) 292 (insert 293 (propertize 294 "All queries resolved successfully.\n\n" 295 'face 'success) 296 " " continue ": Add all feeds to the Elfeed database.\n" 297 " " continue-extra ": Add all feeds, fetch entries from them and open Elfeed.\n" 298 " " copy ": Copy the list of feed URLs as a list\n")) 299 (insert "\n" cancel-q " or " cancel ": Quit and cancel this operation.")) 300 301 (goto-char (point-min)) 302 303 (use-local-map (copy-keymap elfeed-tube-channels-mode-map)) 304 (local-set-key (kbd "C-c C-c") #'elfeed-tube-add--confirm) 305 (local-set-key (kbd "C-c C-w") #'elfeed-tube-add--copy) 306 307 (funcall 308 (if (bound-and-true-p demo-mode) 309 #'switch-to-buffer 310 #'display-buffer) 311 buffer)))) 312 313 (defun elfeed-tube-add--visit-channel (button) 314 "Activate BUTTON." 315 (browse-url (button-get button 'help-echo))) 316 317 ;; (elfeed-tube-add--display-channels my-channels) 318 319 (defun elfeed-tube-add--confirm (&optional arg) 320 "Confirm the addition of visible Youtube feeds to the Elfeed database. 321 322 With optional prefix argument ARG, update these feeds and open Elfeed 323 afterwards." 324 (interactive "P") 325 (cl-assert (derived-mode-p 'elfeed-tube-channels-mode)) 326 (let* ((channels tabulated-list-entries)) 327 (let ((inhibit-message t)) 328 (cl-loop for channel in channels 329 for (_ _ feed) = (append (cadr channel) nil) 330 do (elfeed-add-feed feed :save t))) 331 (message "Added to elfeed-feeds.") 332 (when arg (elfeed)))) 333 334 (defvar elfeed-tube-channels-mode-map 335 (let ((map (make-sparse-keymap))) 336 (define-key map (kbd "C-c C-k") (lambda () (interactive) (quit-window 'kill-buffer))) 337 map)) 338 339 (define-derived-mode elfeed-tube-channels-mode tabulated-list-mode 340 "Elfeed Tube Channels" 341 (setq tabulated-list-use-header-line t ; default to no header 342 ;; tabulated-list--header-string nil 343 header-line-format nil)) 344 345 (defun elfeed-tube-add--copy () 346 "Copy visible Youtube feeds to the kill ring as a list. 347 348 With optional prefix argument ARG, update these feeds and open Elfeed 349 afterwards." 350 (interactive) 351 (cl-assert (derived-mode-p 'elfeed-tube-channels-mode)) 352 (let* ((channels tabulated-list-entries)) 353 (cl-loop for channel in channels 354 for (_ _ feed) = (append (cadr channel) nil) 355 collect feed into feeds 356 finally (kill-new (prin1-to-string feeds))) 357 (message "Feed URLs saved to kill-ring."))) 358 359 (aio-defun elfeed-tube--aio-fetch (url &optional next desc attempts) 360 "Fetch URL asynchronously using `elfeed-curl-retrieve'. 361 362 If successful (HTTP 200), return the JSON-parsed result as a 363 plist. 364 365 Otherwise, call the function NEXT (with no arguments) and try 366 ATTEMPTS more times. Return nil if all attempts fail. DESC is a 367 description string to print to the elfeed-tube log allong with 368 any other error messages. 369 370 This function returns a promise." 371 (let ((attempts (or attempts (1+ elfeed-tube--max-retries)))) 372 (when (> attempts 0) 373 (let* ((response 374 (aio-await (elfeed-tube-curl-enqueue url :method "GET"))) 375 (content (plist-get response :content)) 376 (status (plist-get response :status-code)) 377 (error-msg (plist-get response :error-message))) 378 (cond 379 ((= status 200) 380 (condition-case nil 381 (json-parse-string content :object-type 'plist) 382 ((json-parse-error error) 383 (elfeed-tube-log 'error "[Search] JSON malformed (%s)" 384 (elfeed-tube--attempt-log attempts)) 385 (and (functionp next) (funcall next)) 386 (aio-await 387 (elfeed-tube--aio-fetch url next desc (1- attempts)))))) 388 (t (elfeed-tube-log 'error "[Search][%s]: %s (%s)" error-msg url 389 (elfeed-tube--attempt-log attempts)) 390 (and (functionp next) (funcall next)) 391 (aio-await 392 (elfeed-tube--aio-fetch url next desc (1- attempts))))))))) 393 394 (defun elfeed-tube--entry-create (feed-id entry-data) 395 "Create an Elfeed entry from ENTRY-DATA for feed with id FEED-ID. 396 397 FEED-ID is the id of the feed in the Elfeed database. ENTRY-DATA 398 is a plist of video metadata." 399 (cl-assert (listp entry-data)) 400 (cl-assert (plist-get entry-data :videoId)) 401 402 (let* ((video-id (plist-get entry-data :videoId)) 403 (link (format "https://www.youtube.com/watch?v=%s" video-id)) 404 (title (plist-get entry-data :title)) 405 (published (plist-get entry-data :published)) 406 (author `((:name ,(plist-get entry-data :author) 407 :uri ,feed-id)))) 408 (elfeed-entry--create 409 :link link 410 :title title 411 :id `("www.youtube.com" . ,(concat "yt:video:" video-id)) 412 :date published 413 :tags '(unread) 414 :content-type 'html 415 :meta `(:authors ,author) 416 :feed-id feed-id))) 417 418 (aio-defun elfeed-tube--fake-entry (url &optional force-fetch) 419 (string-match (concat elfeed-tube-youtube-regexp 420 (rx (zero-or-one "watch?v=") 421 (group (1+ (not (or "&" "?")))))) 422 url) 423 (if-let ((video-id (match-string 1 url))) 424 (progn 425 (message "Creating a video summary...") 426 (cl-letf* ((elfeed-show-unique-buffers t) 427 (elfeed-show-entry-switch #'display-buffer) 428 (elfeed-tube-save-indicator nil) 429 (elfeed-tube-auto-save-p nil) 430 (api-data (aio-await 431 (elfeed-tube--aio-fetch 432 (concat (aio-await (elfeed-tube--get-invidious-url)) 433 elfeed-tube--api-videos-path 434 video-id 435 "?fields=" 436 ;; "videoThumbnails,descriptionHtml,lengthSeconds," 437 "title,author,authorUrl,published,videoId") 438 #'elfeed-tube--nrotate-invidious-servers))) 439 (feed-id (concat "https://www.youtube.com/feeds/videos.xml?channel_id=" 440 (nth 1 (split-string (plist-get api-data :authorUrl) 441 "/" t)))) 442 (author `((:name ,(plist-get api-data :author) 443 :uri ,feed-id))) 444 (entry (elfeed-tube--entry-create feed-id api-data)) 445 ((symbol-function 'elfeed-entry-feed) 446 (lambda (_) 447 (elfeed-feed--create 448 :id feed-id 449 :url feed-id 450 :title (plist-get api-data :author) 451 :author author)))) 452 (aio-await (elfeed-tube--fetch-1 entry force-fetch)) 453 (with-selected-window (elfeed-show-entry entry) 454 (message "Summary created for video: \"%s\"" 455 (elfeed-entry-title entry)) 456 (setq-local elfeed-show-refresh-function 457 (lambda () (interactive) 458 (elfeed-tube-show)) 459 elfeed-tube-save-indicator nil)))) 460 (message "Not a youtube video URL, aborting."))) 461 462 (defsubst elfeed-tube--line-at-point () 463 "Get line around point." 464 (buffer-substring (line-beginning-position) (line-end-position))) 465 466 (defun elfeed-tube-next-heading (&optional arg) 467 "Jump to the next heading in an Elfeed entry. 468 469 With numeric prefix argument ARG, jump forward that many times. 470 If ARG is negative, jump backwards instead." 471 (interactive "p") 472 (unless arg (setq arg 1)) 473 (catch 'return 474 (dotimes (_ (abs arg)) 475 (when (> arg 0) (end-of-line)) 476 (if-let ((match 477 (funcall (if (> arg 0) 478 #'text-property-search-forward 479 #'text-property-search-backward) 480 'face `(shr-h1 shr-h2 shr-h3 481 message-header-name elfeed-tube-chapter-face) 482 (lambda (tags face) 483 (cl-loop for x in (if (consp face) face (list face)) 484 thereis (memq x tags))) 485 t))) 486 (goto-char 487 (if (> arg 0) (prop-match-beginning match) (prop-match-end match))) 488 (throw 'return nil)) 489 (when (< arg 0) (beginning-of-line))) 490 (beginning-of-line) 491 (point))) 492 493 (defun elfeed-tube-prev-heading (&optional arg) 494 "Jump to the previous heading in an Elfeed entry. 495 496 With numeric prefix argument ARG, jump backward that many times. 497 If ARG is negative, jump forward instead." 498 (interactive "p") 499 (elfeed-tube-next-heading (- (or arg 1)))) 500 501 (provide 'elfeed-tube-utils) 502 ;;; elfeed-tube-utils.el ends here