diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8f01fa0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,33 @@ + +################################################################################ +# # +# Ｅｄｉｔｏｒｃｏｎｆｉｇ # +# # +# This file defines basic file properties that # +# editors / IDEs use when you modify documents. # +# # +# Check https://EditorConfig.org for details. # +# # +################################################################################ + + +root = true + + +[*] + +end_of_line = lf + +[LICENSE] +[*.{editorconfig,gitignore,json,ts,md}] + +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space +indent_size = 4 +charset = utf-8 + + +[*.md] + +trim_trailing_whitespace = false \ No newline at end of file diff --git a/.gitignore b/.gitignore index caf8035..4ccd04a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -.vscode -video.mp4 -music.mp3 -audio.mp3 +.vscode +video.mp4 +music.mp3 +audio.mp3 diff --git a/LICENSE b/LICENSE index 66ef7c7..28fe267 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ -Copyright 2021 DjDeveloperr - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - +Copyright 2021 DjDeveloperr + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index b881af3..52e15a8 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,26 @@ -# ytdl_core - -Deno port of [ytdl-core](https://www.npmjs.com/package/ytdl-core) using Web Streams API. - -## Usage - -```ts -import ytdl from "https://deno.land/x/ytdl_core/mod.ts"; - -const stream = await ytdl("vRXZj0DzXIA"); - -const chunks: Uint8Array[] = []; - -for await (const chunk of stream) { - chunks.push(chunk); -} - -const blob = new Blob(chunks); -await Deno.writeFile("video.mp4", new Uint8Array(await blob.arrayBuffer())); -``` - -## License - -Check [License](./LICENSE) for more info. - -Copyright 2021 DjDeveloper, Copyright (C) 2012-present by fent +# ytdl_core + +Deno port of [ytdl-core](https://www.npmjs.com/package/ytdl-core) using Web Streams API. + +## Usage + +```ts +import ytdl from "https://deno.land/x/ytdl_core/mod.ts"; + +const stream = await ytdl("vRXZj0DzXIA"); + +const chunks: Uint8Array[] = []; + +for await (const chunk of stream) { + chunks.push(chunk); +} + +const blob = new Blob(chunks); +await Deno.writeFile("video.mp4", new Uint8Array(await blob.arrayBuffer())); +``` + +## License + +Check [License](./LICENSE) for more info. + +Copyright 2021 DjDeveloper, Copyright (C) 2012-present by fent diff --git a/deps.ts b/deps.ts index 7e38f70..0c032cd 100644 --- a/deps.ts +++ b/deps.ts @@ -1,4 +1,3 @@ -export {default as querystring} from "https://esm.sh/querystring@0.2.1"; -export { humanStr as parseTimestamp } from "https://raw.githubusercontent.com/fent/node-m3u8stream/master/src/parse-time.ts"; -export * as sax from "https://deno.land/x/sax_ts@v1.2.10/src/sax.ts"; -export { default as m3u8stream } from "https://esm.sh/m3u8stream@0.8.6"; +export { humanStr as parseTimestamp } from "https://raw.githubusercontent.com/fent/node-m3u8stream/master/src/parse-time.ts"; +export * as sax from "https://deno.land/x/sax_ts@v1.2.10/src/sax.ts"; +export { default as m3u8stream } from "https://esm.sh/m3u8stream@0.8.6"; diff --git a/mod.ts b/mod.ts index a70ec8a..bb6eee8 100644 --- a/mod.ts +++ b/mod.ts @@ -1,15 +1,15 @@ -export * from "./src/cache.ts"; -export * from "./src/format_util.ts"; -export * from "./src/formats.ts"; -export * from "./src/info.ts"; -export * from "./src/info_extras.ts"; -export * from "./src/url_utils.ts"; -export { - decipherFormats, - setDownloadURL, - cache as sigCache, -} from "./src/sig.ts"; -export * from "./src/utils.ts"; -export * from "./src/stream.ts"; -export * from "./src/video.ts"; -export { ytdl as default } from "./src/video.ts"; +export * from "./src/cache.ts"; +export * from "./src/format_util.ts"; +export * from "./src/formats.ts"; +export * from "./src/info.ts"; +export * from "./src/info_extras.ts"; +export * from "./src/url_utils.ts"; +export { + decipherFormats, + setDownloadURL, + cache as sigCache, +} from "./src/sig.ts"; +export * from "./src/utils.ts"; +export * from "./src/stream.ts"; +export * from "./src/video.ts"; +export { ytdl as default } from "./src/video.ts"; diff --git a/src/cache.ts b/src/cache.ts index c24e63e..edbcb52 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,58 +1,58 @@ -export class Cache extends Map { - constructor(public timeout = 1000) { - super(); - } - - set(key: string, value: any) { - if (this.has(key)) { - clearTimeout(super.get(key).tid); - } - - super.set(key, { - tid: setTimeout(this.delete.bind(this, key), this.timeout), - value, - }); - - return this; - } - - get(key: string) { - let entry = super.get(key); - if (entry) { - return entry.value; - } - return null; - } - - getOrSet(key: string, fn: CallableFunction) { - if (this.has(key)) { - return this.get(key); - } else { - let value = fn(); - this.set(key, value); - (async () => { - try { - await value; - } catch (err) { - this.delete(key); - } - })(); - return value; - } - } - - delete(key: string) { - let entry = super.get(key); - if (entry) { - clearTimeout(entry.tid); - return super.delete(key); - } else return false; - } - - clear() { - for (let entry of this.values()) { - clearTimeout(entry.tid); - } - super.clear(); - } -} +export class Cache extends Map { + constructor(public timeout = 1000) { + super(); + } + + set(key: string, value: any) { + if (this.has(key)) { + clearTimeout(super.get(key).tid); + } + + super.set(key, { + tid: setTimeout(this.delete.bind(this, key), this.timeout), + value, + }); + + return this; + } + + get(key: string) { + let entry = super.get(key); + if (entry) { + return entry.value; + } + return null; + } + + getOrSet(key: string, fn: CallableFunction) { + if (this.has(key)) { + return this.get(key); + } else { + let value = fn(); + this.set(key, value); + (async () => { + try { + await value; + } catch (err) { + this.delete(key); + } + })(); + return value; + } + } + + delete(key: string) { + let entry = super.get(key); + if (entry) { + clearTimeout(entry.tid); + return super.delete(key); + } else return false; + } + + clear() { + for (let entry of this.values()) { + clearTimeout(entry.tid); + } + super.clear(); + } +} diff --git a/src/format_util.ts b/src/format_util.ts index 9687330..963a020 100644 --- a/src/format_util.ts +++ b/src/format_util.ts @@ -1,202 +1,202 @@ -import { Format, formats as FORMATS } from "./formats.ts"; -import { ChooseFormatOptions, Filter, VideoFormat } from "./types.ts"; -import * as utils from "./utils.ts"; - -const audioEncodingRanks = ["mp4a", "mp3", "vorbis", "aac", "opus", "flac"]; -const videoEncodingRanks = [ - "mp4v", - "avc1", - "Sorenson H.283", - "MPEG-4 Visual", - "VP8", - "VP9", - "H.264", -]; - -const getVideoBitrate = (format: Format) => format.bitrate ?? 0; -const getVideoEncodingRank = (format: any) => - videoEncodingRanks.findIndex( - (enc) => format.codecs && format.codecs.includes(enc) - ); -const getAudioBitrate = (format: Format) => format.audioBitrate || 0; -const getAudioEncodingRank = (format: any) => - audioEncodingRanks.findIndex( - (enc) => format.codecs && format.codecs.includes(enc) - ); - -/** - * Sort formats by a list of functions. - */ -const sortFormatsBy = (a: any, b: any, sortBy: CallableFunction[]) => { - let res = 0; - for (let fn of sortBy) { - res = fn(b) - fn(a); - if (res !== 0) { - break; - } - } - return res; -}; - -const sortFormatsByVideo = (a: any, b: any) => - sortFormatsBy(a, b, [ - (format: any) => parseInt(format.qualityLabel), - getVideoBitrate, - getVideoEncodingRank, - ]); - -const sortFormatsByAudio = (a: any, b: any) => - sortFormatsBy(a, b, [getAudioBitrate, getAudioEncodingRank]); - -export const sortFormats = (a: any, b: any) => - sortFormatsBy(a, b, [ - // Formats with both video and audio are ranked highest. - (format: any) => +!!format.isHLS, - (format: any) => +!!format.isDashMPD, - (format: any) => +(format.contentLength > 0), - (format: any) => +(format.hasVideo && format.hasAudio), - (format: any) => +format.hasVideo, - (format: any) => parseInt(format.qualityLabel) || 0, - getVideoBitrate, - getAudioBitrate, - getVideoEncodingRank, - getAudioEncodingRank, - ]); - -export function chooseFormat( - formats: VideoFormat[], - options: ChooseFormatOptions -) { - if (typeof options.format === "object") { - if (!options.format.url) { - throw Error("Invalid format given, did you use `ytdl.getInfo()`?"); - } - return options.format; - } - - if (options.filter) { - formats = filterFormats(formats, options.filter as any); - } - - let format; - const quality = options.quality || "highest"; - switch (quality) { - case "highest": - format = formats[0]; - break; - - case "lowest": - format = formats[formats.length - 1]; - break; - - case "highestaudio": - formats = filterFormats(formats, "audio"); - formats.sort(sortFormatsByAudio); - format = formats[0]; - break; - - case "lowestaudio": - formats = filterFormats(formats, "audio"); - formats.sort(sortFormatsByAudio); - format = formats[formats.length - 1]; - break; - - case "highestvideo": - formats = filterFormats(formats, "video"); - formats.sort(sortFormatsByVideo); - format = formats[0]; - break; - - case "lowestvideo": - formats = filterFormats(formats, "video"); - formats.sort(sortFormatsByVideo); - format = formats[formats.length - 1]; - break; - - default: - format = getFormatByQuality( - Array.isArray(quality) - ? quality.map((e: number | string) => e.toString()) - : quality.toString(), - formats - ); - break; - } - - if (!format) { - throw Error(`No such format found: ${quality}`); - } - return format; -} - -/** - * Gets a format based on quality or array of quality's - */ -const getFormatByQuality = ( - quality: string | string[], - formats: VideoFormat[] -) => { - let getFormat = (itag: any) => - formats.find((format) => `${format.itag}` === `${itag}`); - if (Array.isArray(quality)) { - return getFormat(quality.find((q) => getFormat(q))); - } else { - return getFormat(quality); - } -}; - -export function filterFormats(formats: VideoFormat[], filter: Filter) { - let fn: (format: VideoFormat) => boolean; - switch (filter) { - case "videoandaudio": - case "audioandvideo": - fn = (format) => format.hasVideo && format.hasAudio; - break; - - case "video": - fn = (format) => format.hasVideo; - break; - - case "videoonly": - fn = (format) => format.hasVideo && !format.hasAudio; - break; - - case "audio": - fn = (format) => format.hasAudio; - break; - - case "audioonly": - fn = (format) => !format.hasVideo && format.hasAudio; - break; - - default: - if (typeof filter === "function") { - fn = filter; - } else { - throw TypeError(`Given filter (${filter}) is not supported`); - } - } - return formats.filter((format) => !!format.url && fn(format)); -} - -export function addFormatMeta(format: VideoFormat) { - format = Object.assign({}, FORMATS[format.itag], format); - format.hasVideo = !!format.qualityLabel; - format.hasAudio = !!format.audioBitrate; - format.container = (format.mimeType - ? format.mimeType.split(";")[0].split("/")[1] - : null) as any; - format.codecs = format.mimeType - ? utils.between(format.mimeType, 'codecs="', '"') - : null!; - format.videoCodec = - format.hasVideo && format.codecs ? format.codecs.split(", ")[0] : null!; - format.audioCodec = - format.hasAudio && format.codecs - ? format.codecs.split(", ").slice(-1)[0] - : null!; - format.isLive = /\bsource[/=]yt_live_broadcast\b/.test(format.url); - format.isHLS = /\/manifest\/hls_(variant|playlist)\//.test(format.url); - format.isDashMPD = /\/manifest\/dash\//.test(format.url); - return format; -} +import { Format, formats as FORMATS } from "./formats.ts"; +import { ChooseFormatOptions, Filter, VideoFormat } from "./types.ts"; +import * as utils from "./utils.ts"; + +const audioEncodingRanks = ["mp4a", "mp3", "vorbis", "aac", "opus", "flac"]; +const videoEncodingRanks = [ + "mp4v", + "avc1", + "Sorenson H.283", + "MPEG-4 Visual", + "VP8", + "VP9", + "H.264", +]; + +const getVideoBitrate = (format: Format) => format.bitrate ?? 0; +const getVideoEncodingRank = (format: any) => + videoEncodingRanks.findIndex( + (enc) => format.codecs && format.codecs.includes(enc) + ); +const getAudioBitrate = (format: Format) => format.audioBitrate || 0; +const getAudioEncodingRank = (format: any) => + audioEncodingRanks.findIndex( + (enc) => format.codecs && format.codecs.includes(enc) + ); + +/** + * Sort formats by a list of functions. + */ +const sortFormatsBy = (a: any, b: any, sortBy: CallableFunction[]) => { + let res = 0; + for (let fn of sortBy) { + res = fn(b) - fn(a); + if (res !== 0) { + break; + } + } + return res; +}; + +const sortFormatsByVideo = (a: any, b: any) => + sortFormatsBy(a, b, [ + (format: any) => parseInt(format.qualityLabel), + getVideoBitrate, + getVideoEncodingRank, + ]); + +const sortFormatsByAudio = (a: any, b: any) => + sortFormatsBy(a, b, [getAudioBitrate, getAudioEncodingRank]); + +export const sortFormats = (a: any, b: any) => + sortFormatsBy(a, b, [ + // Formats with both video and audio are ranked highest. + (format: any) => +!!format.isHLS, + (format: any) => +!!format.isDashMPD, + (format: any) => +(format.contentLength > 0), + (format: any) => +(format.hasVideo && format.hasAudio), + (format: any) => +format.hasVideo, + (format: any) => parseInt(format.qualityLabel) || 0, + getVideoBitrate, + getAudioBitrate, + getVideoEncodingRank, + getAudioEncodingRank, + ]); + +export function chooseFormat( + formats: VideoFormat[], + options: ChooseFormatOptions +) { + if (typeof options.format === "object") { + if (!options.format.url) { + throw Error("Invalid format given, did you use `ytdl.getInfo()`?"); + } + return options.format; + } + + if (options.filter) { + formats = filterFormats(formats, options.filter as any); + } + + let format; + const quality = options.quality || "highest"; + switch (quality) { + case "highest": + format = formats[0]; + break; + + case "lowest": + format = formats[formats.length - 1]; + break; + + case "highestaudio": + formats = filterFormats(formats, "audio"); + formats.sort(sortFormatsByAudio); + format = formats[0]; + break; + + case "lowestaudio": + formats = filterFormats(formats, "audio"); + formats.sort(sortFormatsByAudio); + format = formats[formats.length - 1]; + break; + + case "highestvideo": + formats = filterFormats(formats, "video"); + formats.sort(sortFormatsByVideo); + format = formats[0]; + break; + + case "lowestvideo": + formats = filterFormats(formats, "video"); + formats.sort(sortFormatsByVideo); + format = formats[formats.length - 1]; + break; + + default: + format = getFormatByQuality( + Array.isArray(quality) + ? quality.map((e: number | string) => e.toString()) + : quality.toString(), + formats + ); + break; + } + + if (!format) { + throw Error(`No such format found: ${quality}`); + } + return format; +} + +/** + * Gets a format based on quality or array of quality's + */ +const getFormatByQuality = ( + quality: string | string[], + formats: VideoFormat[] +) => { + let getFormat = (itag: any) => + formats.find((format) => `${format.itag}` === `${itag}`); + if (Array.isArray(quality)) { + return getFormat(quality.find((q) => getFormat(q))); + } else { + return getFormat(quality); + } +}; + +export function filterFormats(formats: VideoFormat[], filter: Filter) { + let fn: (format: VideoFormat) => boolean; + switch (filter) { + case "videoandaudio": + case "audioandvideo": + fn = (format) => format.hasVideo && format.hasAudio; + break; + + case "video": + fn = (format) => format.hasVideo; + break; + + case "videoonly": + fn = (format) => format.hasVideo && !format.hasAudio; + break; + + case "audio": + fn = (format) => format.hasAudio; + break; + + case "audioonly": + fn = (format) => !format.hasVideo && format.hasAudio; + break; + + default: + if (typeof filter === "function") { + fn = filter; + } else { + throw TypeError(`Given filter (${filter}) is not supported`); + } + } + return formats.filter((format) => !!format.url && fn(format)); +} + +export function addFormatMeta(format: VideoFormat) { + format = Object.assign({}, FORMATS[format.itag], format); + format.hasVideo = !!format.qualityLabel; + format.hasAudio = !!format.audioBitrate; + format.container = (format.mimeType + ? format.mimeType.split(";")[0].split("/")[1] + : null) as any; + format.codecs = format.mimeType + ? utils.between(format.mimeType, 'codecs="', '"') + : null!; + format.videoCodec = + format.hasVideo && format.codecs ? format.codecs.split(", ")[0] : null!; + format.audioCodec = + format.hasAudio && format.codecs + ? format.codecs.split(", ").slice(-1)[0] + : null!; + format.isLive = /\bsource[/=]yt_live_broadcast\b/.test(format.url); + format.isHLS = /\/manifest\/hls_(variant|playlist)\//.test(format.url); + format.isDashMPD = /\/manifest\/dash\//.test(format.url); + return format; +} diff --git a/src/formats.ts b/src/formats.ts index 6d6bc16..476ad8a 100644 --- a/src/formats.ts +++ b/src/formats.ts @@ -1,528 +1,528 @@ -export interface Format { - mimeType: string; - qualityLabel: string | null; - bitrate: number | null; - audioBitrate: number | null; -} - -export const formats: { - [name: string]: Format; -} = { - 5: { - mimeType: 'video/flv; codecs="Sorenson H.283, mp3"', - qualityLabel: "240p", - bitrate: 250000, - audioBitrate: 64, - }, - - 6: { - mimeType: 'video/flv; codecs="Sorenson H.263, mp3"', - qualityLabel: "270p", - bitrate: 800000, - audioBitrate: 64, - }, - - 13: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: null, - bitrate: 500000, - audioBitrate: null, - }, - - 17: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: "144p", - bitrate: 50000, - audioBitrate: 24, - }, - - 18: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "360p", - bitrate: 500000, - audioBitrate: 96, - }, - - 22: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 2000000, - audioBitrate: 192, - }, - - 34: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: "360p", - bitrate: 500000, - audioBitrate: 128, - }, - - 35: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: "480p", - bitrate: 800000, - audioBitrate: 128, - }, - - 36: { - mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', - qualityLabel: "240p", - bitrate: 175000, - audioBitrate: 32, - }, - - 37: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "1080p", - bitrate: 3000000, - audioBitrate: 192, - }, - - 38: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "3072p", - bitrate: 3500000, - audioBitrate: 192, - }, - - 43: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: "360p", - bitrate: 500000, - audioBitrate: 128, - }, - - 44: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: "480p", - bitrate: 1000000, - audioBitrate: 128, - }, - - 45: { - mimeType: 'video/webm; codecs="VP8, vorbis"', - qualityLabel: "720p", - bitrate: 2000000, - audioBitrate: 192, - }, - - 46: { - mimeType: 'audio/webm; codecs="vp8, vorbis"', - qualityLabel: "1080p", - bitrate: null, - audioBitrate: 192, - }, - - 82: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "360p", - bitrate: 500000, - audioBitrate: 96, - }, - - 83: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "240p", - bitrate: 500000, - audioBitrate: 96, - }, - - 84: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 2000000, - audioBitrate: 192, - }, - - 85: { - mimeType: 'video/mp4; codecs="H.264, aac"', - qualityLabel: "1080p", - bitrate: 3000000, - audioBitrate: 192, - }, - - 91: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "144p", - bitrate: 100000, - audioBitrate: 48, - }, - - 92: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "240p", - bitrate: 150000, - audioBitrate: 48, - }, - - 93: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "360p", - bitrate: 500000, - audioBitrate: 128, - }, - - 94: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "480p", - bitrate: 800000, - audioBitrate: 128, - }, - - 95: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 1500000, - audioBitrate: 256, - }, - - 96: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "1080p", - bitrate: 2500000, - audioBitrate: 256, - }, - - 100: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: "360p", - bitrate: null, - audioBitrate: 128, - }, - - 101: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: "360p", - bitrate: null, - audioBitrate: 192, - }, - - 102: { - mimeType: 'audio/webm; codecs="VP8, vorbis"', - qualityLabel: "720p", - bitrate: null, - audioBitrate: 192, - }, - - 120: { - mimeType: 'video/flv; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 2000000, - audioBitrate: 128, - }, - - 127: { - mimeType: 'audio/ts; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 96, - }, - - 128: { - mimeType: 'audio/ts; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 96, - }, - - 132: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "240p", - bitrate: 150000, - audioBitrate: 48, - }, - - 133: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "240p", - bitrate: 200000, - audioBitrate: null, - }, - - 134: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "360p", - bitrate: 300000, - audioBitrate: null, - }, - - 135: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "480p", - bitrate: 500000, - audioBitrate: null, - }, - - 136: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "720p", - bitrate: 1000000, - audioBitrate: null, - }, - - 137: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "1080p", - bitrate: 2500000, - audioBitrate: null, - }, - - 138: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "4320p", - bitrate: 13500000, - audioBitrate: null, - }, - - 139: { - mimeType: 'audio/mp4; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 48, - }, - - 140: { - mimeType: 'audio/m4a; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 128, - }, - - 141: { - mimeType: 'audio/mp4; codecs="aac"', - qualityLabel: null, - bitrate: null, - audioBitrate: 256, - }, - - 151: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 50000, - audioBitrate: 24, - }, - - 160: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "144p", - bitrate: 100000, - audioBitrate: null, - }, - - 171: { - mimeType: 'audio/webm; codecs="vorbis"', - qualityLabel: null, - bitrate: null, - audioBitrate: 128, - }, - - 172: { - mimeType: 'audio/webm; codecs="vorbis"', - qualityLabel: null, - bitrate: null, - audioBitrate: 192, - }, - - 242: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "240p", - bitrate: 100000, - audioBitrate: null, - }, - - 243: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "360p", - bitrate: 250000, - audioBitrate: null, - }, - - 244: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "480p", - bitrate: 500000, - audioBitrate: null, - }, - - 247: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "720p", - bitrate: 700000, - audioBitrate: null, - }, - - 248: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1080p", - bitrate: 1500000, - audioBitrate: null, - }, - - 249: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 48, - }, - - 250: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 64, - }, - - 251: { - mimeType: 'audio/webm; codecs="opus"', - qualityLabel: null, - bitrate: null, - audioBitrate: 160, - }, - - 264: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "1440p", - bitrate: 4000000, - audioBitrate: null, - }, - - 266: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "2160p", - bitrate: 12500000, - audioBitrate: null, - }, - - 271: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1440p", - bitrate: 9000000, - audioBitrate: null, - }, - - 272: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "4320p", - bitrate: 20000000, - audioBitrate: null, - }, - - 278: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "144p 30fps", - bitrate: 80000, - audioBitrate: null, - }, - - 298: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "720p", - bitrate: 3000000, - audioBitrate: null, - }, - - 299: { - mimeType: 'video/mp4; codecs="H.264"', - qualityLabel: "1080p", - bitrate: 5500000, - audioBitrate: null, - }, - - 300: { - mimeType: 'video/ts; codecs="H.264, aac"', - qualityLabel: "720p", - bitrate: 1318000, - audioBitrate: 48, - }, - - 302: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "720p HFR", - bitrate: 2500000, - audioBitrate: null, - }, - - 303: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1080p HFR", - bitrate: 5000000, - audioBitrate: null, - }, - - 308: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1440p HFR", - bitrate: 10000000, - audioBitrate: null, - }, - - 313: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "2160p", - bitrate: 13000000, - audioBitrate: null, - }, - - 315: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "2160p HFR", - bitrate: 20000000, - audioBitrate: null, - }, - - 330: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "144p HDR, HFR", - bitrate: 80000, - audioBitrate: null, - }, - - 331: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "240p HDR, HFR", - bitrate: 100000, - audioBitrate: null, - }, - - 332: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "360p HDR, HFR", - bitrate: 250000, - audioBitrate: null, - }, - - 333: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "240p HDR, HFR", - bitrate: 500000, - audioBitrate: null, - }, - - 334: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "720p HDR, HFR", - bitrate: 1000000, - audioBitrate: null, - }, - - 335: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1080p HDR, HFR", - bitrate: 1500000, - audioBitrate: null, - }, - - 336: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "1440p HDR, HFR", - bitrate: 5000000, - audioBitrate: null, - }, - - 337: { - mimeType: 'video/webm; codecs="VP9"', - qualityLabel: "2160p HDR, HFR", - bitrate: 12000000, - audioBitrate: null, - }, -}; +export interface Format { + mimeType: string; + qualityLabel: string | null; + bitrate: number | null; + audioBitrate: number | null; +} + +export const formats: { + [name: string]: Format; +} = { + 5: { + mimeType: 'video/flv; codecs="Sorenson H.283, mp3"', + qualityLabel: "240p", + bitrate: 250000, + audioBitrate: 64, + }, + + 6: { + mimeType: 'video/flv; codecs="Sorenson H.263, mp3"', + qualityLabel: "270p", + bitrate: 800000, + audioBitrate: 64, + }, + + 13: { + mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', + qualityLabel: null, + bitrate: 500000, + audioBitrate: null, + }, + + 17: { + mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', + qualityLabel: "144p", + bitrate: 50000, + audioBitrate: 24, + }, + + 18: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "360p", + bitrate: 500000, + audioBitrate: 96, + }, + + 22: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 2000000, + audioBitrate: 192, + }, + + 34: { + mimeType: 'video/flv; codecs="H.264, aac"', + qualityLabel: "360p", + bitrate: 500000, + audioBitrate: 128, + }, + + 35: { + mimeType: 'video/flv; codecs="H.264, aac"', + qualityLabel: "480p", + bitrate: 800000, + audioBitrate: 128, + }, + + 36: { + mimeType: 'video/3gp; codecs="MPEG-4 Visual, aac"', + qualityLabel: "240p", + bitrate: 175000, + audioBitrate: 32, + }, + + 37: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "1080p", + bitrate: 3000000, + audioBitrate: 192, + }, + + 38: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "3072p", + bitrate: 3500000, + audioBitrate: 192, + }, + + 43: { + mimeType: 'video/webm; codecs="VP8, vorbis"', + qualityLabel: "360p", + bitrate: 500000, + audioBitrate: 128, + }, + + 44: { + mimeType: 'video/webm; codecs="VP8, vorbis"', + qualityLabel: "480p", + bitrate: 1000000, + audioBitrate: 128, + }, + + 45: { + mimeType: 'video/webm; codecs="VP8, vorbis"', + qualityLabel: "720p", + bitrate: 2000000, + audioBitrate: 192, + }, + + 46: { + mimeType: 'audio/webm; codecs="vp8, vorbis"', + qualityLabel: "1080p", + bitrate: null, + audioBitrate: 192, + }, + + 82: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "360p", + bitrate: 500000, + audioBitrate: 96, + }, + + 83: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "240p", + bitrate: 500000, + audioBitrate: 96, + }, + + 84: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 2000000, + audioBitrate: 192, + }, + + 85: { + mimeType: 'video/mp4; codecs="H.264, aac"', + qualityLabel: "1080p", + bitrate: 3000000, + audioBitrate: 192, + }, + + 91: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "144p", + bitrate: 100000, + audioBitrate: 48, + }, + + 92: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "240p", + bitrate: 150000, + audioBitrate: 48, + }, + + 93: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "360p", + bitrate: 500000, + audioBitrate: 128, + }, + + 94: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "480p", + bitrate: 800000, + audioBitrate: 128, + }, + + 95: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 1500000, + audioBitrate: 256, + }, + + 96: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "1080p", + bitrate: 2500000, + audioBitrate: 256, + }, + + 100: { + mimeType: 'audio/webm; codecs="VP8, vorbis"', + qualityLabel: "360p", + bitrate: null, + audioBitrate: 128, + }, + + 101: { + mimeType: 'audio/webm; codecs="VP8, vorbis"', + qualityLabel: "360p", + bitrate: null, + audioBitrate: 192, + }, + + 102: { + mimeType: 'audio/webm; codecs="VP8, vorbis"', + qualityLabel: "720p", + bitrate: null, + audioBitrate: 192, + }, + + 120: { + mimeType: 'video/flv; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 2000000, + audioBitrate: 128, + }, + + 127: { + mimeType: 'audio/ts; codecs="aac"', + qualityLabel: null, + bitrate: null, + audioBitrate: 96, + }, + + 128: { + mimeType: 'audio/ts; codecs="aac"', + qualityLabel: null, + bitrate: null, + audioBitrate: 96, + }, + + 132: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "240p", + bitrate: 150000, + audioBitrate: 48, + }, + + 133: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "240p", + bitrate: 200000, + audioBitrate: null, + }, + + 134: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "360p", + bitrate: 300000, + audioBitrate: null, + }, + + 135: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "480p", + bitrate: 500000, + audioBitrate: null, + }, + + 136: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "720p", + bitrate: 1000000, + audioBitrate: null, + }, + + 137: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "1080p", + bitrate: 2500000, + audioBitrate: null, + }, + + 138: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "4320p", + bitrate: 13500000, + audioBitrate: null, + }, + + 139: { + mimeType: 'audio/mp4; codecs="aac"', + qualityLabel: null, + bitrate: null, + audioBitrate: 48, + }, + + 140: { + mimeType: 'audio/m4a; codecs="aac"', + qualityLabel: null, + bitrate: null, + audioBitrate: 128, + }, + + 141: { + mimeType: 'audio/mp4; codecs="aac"', + qualityLabel: null, + bitrate: null, + audioBitrate: 256, + }, + + 151: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 50000, + audioBitrate: 24, + }, + + 160: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "144p", + bitrate: 100000, + audioBitrate: null, + }, + + 171: { + mimeType: 'audio/webm; codecs="vorbis"', + qualityLabel: null, + bitrate: null, + audioBitrate: 128, + }, + + 172: { + mimeType: 'audio/webm; codecs="vorbis"', + qualityLabel: null, + bitrate: null, + audioBitrate: 192, + }, + + 242: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "240p", + bitrate: 100000, + audioBitrate: null, + }, + + 243: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "360p", + bitrate: 250000, + audioBitrate: null, + }, + + 244: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "480p", + bitrate: 500000, + audioBitrate: null, + }, + + 247: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "720p", + bitrate: 700000, + audioBitrate: null, + }, + + 248: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1080p", + bitrate: 1500000, + audioBitrate: null, + }, + + 249: { + mimeType: 'audio/webm; codecs="opus"', + qualityLabel: null, + bitrate: null, + audioBitrate: 48, + }, + + 250: { + mimeType: 'audio/webm; codecs="opus"', + qualityLabel: null, + bitrate: null, + audioBitrate: 64, + }, + + 251: { + mimeType: 'audio/webm; codecs="opus"', + qualityLabel: null, + bitrate: null, + audioBitrate: 160, + }, + + 264: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "1440p", + bitrate: 4000000, + audioBitrate: null, + }, + + 266: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "2160p", + bitrate: 12500000, + audioBitrate: null, + }, + + 271: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1440p", + bitrate: 9000000, + audioBitrate: null, + }, + + 272: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "4320p", + bitrate: 20000000, + audioBitrate: null, + }, + + 278: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "144p 30fps", + bitrate: 80000, + audioBitrate: null, + }, + + 298: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "720p", + bitrate: 3000000, + audioBitrate: null, + }, + + 299: { + mimeType: 'video/mp4; codecs="H.264"', + qualityLabel: "1080p", + bitrate: 5500000, + audioBitrate: null, + }, + + 300: { + mimeType: 'video/ts; codecs="H.264, aac"', + qualityLabel: "720p", + bitrate: 1318000, + audioBitrate: 48, + }, + + 302: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "720p HFR", + bitrate: 2500000, + audioBitrate: null, + }, + + 303: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1080p HFR", + bitrate: 5000000, + audioBitrate: null, + }, + + 308: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1440p HFR", + bitrate: 10000000, + audioBitrate: null, + }, + + 313: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "2160p", + bitrate: 13000000, + audioBitrate: null, + }, + + 315: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "2160p HFR", + bitrate: 20000000, + audioBitrate: null, + }, + + 330: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "144p HDR, HFR", + bitrate: 80000, + audioBitrate: null, + }, + + 331: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "240p HDR, HFR", + bitrate: 100000, + audioBitrate: null, + }, + + 332: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "360p HDR, HFR", + bitrate: 250000, + audioBitrate: null, + }, + + 333: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "240p HDR, HFR", + bitrate: 500000, + audioBitrate: null, + }, + + 334: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "720p HDR, HFR", + bitrate: 1000000, + audioBitrate: null, + }, + + 335: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1080p HDR, HFR", + bitrate: 1500000, + audioBitrate: null, + }, + + 336: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "1440p HDR, HFR", + bitrate: 5000000, + audioBitrate: null, + }, + + 337: { + mimeType: 'video/webm; codecs="VP9"', + qualityLabel: "2160p HDR, HFR", + bitrate: 12000000, + audioBitrate: null, + }, +}; diff --git a/src/info.ts b/src/info.ts index a22a624..6bb97b9 100644 --- a/src/info.ts +++ b/src/info.ts @@ -1,554 +1,554 @@ -import { querystring, sax } from "../deps.ts"; -import { Cache } from "./cache.ts"; -import * as sig from "./sig.ts"; -import * as urlUtils from "./url_utils.ts"; -import * as utils from "./utils.ts"; -import * as formatUtils from "./format_util.ts"; -import * as extras from "./info_extras.ts"; -import { GetInfoOptions, VideoInfo } from "./types.ts"; -import { request } from "./request.ts"; - -let cver = "2.20210622.10.00"; - -const BASE_URL = "https://www.youtube.com/watch?v="; - -export const cache = new Cache(); -export const cookieCache = new Cache(1000 * 60 * 60 * 24); -export const watchPageCache = new Cache(); - -export class UnrecoverableError extends Error { - name = "UnrecoverableError"; -} - -export const USER_AGENT = - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Safari/537.36"; - -const AGE_RESTRICTED_URLS = [ - "support.google.com/youtube/?p=age_restrictions", - "youtube.com/t/community_guidelines", -]; - -export async function getBasicInfo( - id: string, - options: GetInfoOptions = {}, -): Promise { - id = urlUtils.getVideoID(id); - options.headers = Object.assign( - {}, - { - // eslint-disable-next-line max-len - "User-Agent": USER_AGENT, - }, - options.headers, - ); - const validate = (info: any) => { - let playErr = utils.playError( - info.player_response, - ["ERROR"], - UnrecoverableError as any, - ); - let privateErr = privateVideoError(info.player_response); - if (playErr || privateErr) { - throw playErr || privateErr; - } - return ( - info && - info.player_response && - (info.player_response.streamingData || - isRental(info.player_response) || - isNotYetBroadcasted(info.player_response)) - ); - }; - let info = await pipeline([id, options], validate, {}, [ - getWatchHTMLPage, - getWatchJSONPage, - getVideoInfoPage, - ]); - - if (info.player_response === undefined) throw new Error("404 - Not found"); - - Object.assign(info, { - formats: parseFormats(info.player_response), - related_videos: extras.getRelatedVideos(info), - }); - - // Add additional properties to info. - const media = extras.getMedia(info); - let additional = { - author: extras.getAuthor(info), - media, - likes: extras.getLikes(info), - dislikes: extras.getDislikes(info), - age_restricted: !!( - media && - media.notice_url && - AGE_RESTRICTED_URLS.some((url) => media.notice_url.includes(url)) - ), - - // Give the standard link to the video. - video_url: BASE_URL + id, - storyboards: extras.getStoryboards(info), - }; - - info.videoDetails = extras.cleanVideoDetails( - Object.assign( - {}, - info.player_response && - info.player_response.microformat && - info.player_response.microformat.playerMicroformatRenderer, - info.player_response && info.player_response.videoDetails, - additional, - ), - info, - ); - - return info; -} - -const privateVideoError = (player_response: any) => { - let playability = player_response && player_response.playabilityStatus; - if ( - playability && - playability.status === "LOGIN_REQUIRED" && - playability.messages && - playability.messages.filter((m: any) => /This is a private video/.test(m)) - .length - ) { - return new UnrecoverableError( - playability.reason || (playability.messages && playability.messages[0]), - ); - } else { - return null; - } -}; - -const isRental = (player_response: any) => { - let playability = player_response.playabilityStatus; - return ( - playability && - playability.status === "UNPLAYABLE" && - playability.errorScreen && - playability.errorScreen.playerLegacyDesktopYpcOfferRenderer - ); -}; - -const isNotYetBroadcasted = (player_response: any) => { - let playability = player_response.playabilityStatus; - return playability && playability.status === "LIVE_STREAM_OFFLINE"; -}; - -const getWatchHTMLURL = (id: string, options: any) => - `${BASE_URL + id}&hl=${options.lang || "en"}`; -const getWatchHTMLPageBody = (id: string, options: any) => { - const url = getWatchHTMLURL(id, options); - return watchPageCache.getOrSet(url, () => - request(url, options) - .then((r) => r.text()) - .then((t) => { - return t; - })); -}; - -const EMBED_URL = "https://www.youtube.com/embed/"; -const getEmbedPageBody = (id: string, options: GetInfoOptions = {}) => { - const embedUrl = `${EMBED_URL + id}?hl=${options.lang || "en"}`; - return request(embedUrl, options).then((e) => e.text()); -}; - -const getHTML5player = (body: string) => { - let html5playerRes = - /|"jsUrl":"([^"]+)"/ - .exec( - body, - ); - return html5playerRes ? html5playerRes[1] || html5playerRes[2] : null; -}; - -const getIdentityToken = ( - id: string, - options: any, - key: string, - throwIfNotFound: boolean, -) => - cookieCache.getOrSet(key, async () => { - let page = await getWatchHTMLPageBody(id, options); - let match = page.match(/(["'])ID_TOKEN\1[:,]\s?"([^"]+)"/); - if (!match && throwIfNotFound) { - throw new UnrecoverableError( - "Cookie header used in request, but unable to find YouTube identity token", - ); - } - return match && match[2]; - }); - -/** - * Goes through each endpoint in the pipeline, retrying on failure if the error is recoverable. - * If unable to succeed with one endpoint, moves onto the next one. - */ -const pipeline = async ( - args: any[], - validate: CallableFunction, - retryOptions: any, - endpoints: CallableFunction[], -) => { - let info; - for (let func of endpoints) { - try { - const newInfo = await retryFunc(func, args.concat([info]), retryOptions); - if (newInfo && newInfo.player_response) { - newInfo.player_response.videoDetails = assign( - info && info.player_response && info.player_response.videoDetails, - newInfo.player_response.videoDetails, - ); - newInfo.player_response = assign( - info && info.player_response, - newInfo.player_response, - ); - } - info = assign(info, newInfo); - if (validate(info, false)) { - break; - } - } catch (err) { - if ( - err instanceof UnrecoverableError || - func === endpoints[endpoints.length - 1] - ) { - throw err; - } - // Unable to find video metadata... so try next endpoint. - } - } - return info; -}; - -/** - * Like Object.assign(), but ignores `null` and `undefined` from `source`. - * - * @param {Object} target - * @param {Object} source - * @returns {Object} - */ -const assign = (target: any, source: any) => { - if (!target || !source) { - return target || source; - } - for (let [key, value] of Object.entries(source)) { - if (value !== null && value !== undefined) { - target[key] = value; - } - } - return target; -}; - -/** - * Given a function, calls it with `args` until it's successful, - * or until it encounters an unrecoverable error. - * Currently, any error from miniget is considered unrecoverable. Errors such as - * too many redirects, invalid URL, status code 404, status code 502. - * - * @param {Function} func - * @param {Array.} args - * @param {Object} options - * @param {number} options.maxRetries - * @param {Object} options.backoff - * @param {number} options.backoff.inc - */ -const retryFunc = async (func: CallableFunction, args: any[], options: any) => { - let currentTry = 0, - result; - while (currentTry <= (options.maxRetries ?? 1)) { - try { - result = await func(...args); - break; - } catch (err) { - if ( - err instanceof UnrecoverableError || - err instanceof TypeError || - err.statusCode < 500 || - currentTry >= options.maxRetries - ) { - throw err; - } - let wait = Math.min( - ++currentTry * (options.backoff?.inc ?? 0), - options.backoff?.max ?? 0, - ); - await new Promise((resolve) => setTimeout(resolve, wait)); - } - } - return result; -}; - -const jsonClosingChars = /^[)\]}'\s]+/; -const parseJSON = (source: any, varName: any, json: any) => { - if (!json || typeof json === "object") { - return json; - } else { - try { - json = json.replace(jsonClosingChars, ""); - return JSON.parse(json); - } catch (err) { - throw Error(`Error parsing ${varName} in ${source}: ${err.message}`); - } - } -}; - -const findJSON = ( - source: any, - varName: string, - body: any, - left: any, - right: any, - prependJSON: any, -) => { - let jsonStr = utils.between(body, left, right); - if (!jsonStr) { - throw Error(`Could not find ${varName} in ${source}`); - } - return parseJSON( - source, - varName, - utils.cutAfterJSON(`${prependJSON}${jsonStr}`), - ); -}; - -const findPlayerResponse = (source: any, info: any) => { - const player_response = info && - ((info.args && info.args.player_response) || - info.player_response || - info.playerResponse || - info.embedded_player_response); - return parseJSON(source, "player_response", player_response); -}; - -const getWatchJSONURL = (id: string, options: GetInfoOptions) => - `${getWatchHTMLURL(id, options)}&pbj=1`; -const getWatchJSONPage = async (id: string, options: GetInfoOptions) => { - const reqOptions = Object.assign({ headers: {} }, options); - let cookie = (reqOptions.headers as any).Cookie || - (reqOptions.headers as any).cookie; - reqOptions.headers = Object.assign( - { - "x-youtube-client-name": "1", - "x-youtube-client-version": cver, - "x-youtube-identity-token": cookieCache.get(cookie || "browser") || "", - }, - reqOptions.headers, - ); - - const setIdentityToken = async (key: string, throwIfNotFound: boolean) => { - if ((reqOptions.headers as any)["x-youtube-identity-token"]) { - return; - } - (reqOptions.headers as any)[ - "x-youtube-identity-token" - ] = await getIdentityToken(id, options, key, throwIfNotFound); - }; - - if (cookie) { - await setIdentityToken(cookie, true); - } - - const jsonUrl = getWatchJSONURL(id, options); - let body = await request(jsonUrl, reqOptions).then((e) => e.text()); - let parsedBody = parseJSON("watch.json", "body", body); - if (parsedBody.reload === "now") { - await setIdentityToken("browser", false); - } - if (parsedBody.reload === "now" || !Array.isArray(parsedBody)) { - throw Error("Unable to retrieve video metadata in watch.json"); - } - let info = parsedBody.reduce((part, curr) => Object.assign(curr, part), {}); - info.player_response = findPlayerResponse("watch.json", info); - info.html5player = info.player && info.player.assets && info.player.assets.js; - - return info; -}; - -const getWatchHTMLPage = async (id: string, options: GetInfoOptions) => { - let body = await getWatchHTMLPageBody(id, options); - let info: any = { page: "watch" }; - try { - cver = utils.between(body, '{"key":"cver","value":"', '"}'); - info.player_response = findJSON( - "watch.html", - "player_response", - body, - /\bytInitialPlayerResponse\s*=\s*\{/i, - "", - "{", - ); - } catch (err) { - let args = findJSON( - "watch.html", - "player_response", - body, - /\bytplayer\.config\s*=\s*{/, - "", - "{", - ); - info.player_response = findPlayerResponse("watch.html", args); - } - info.response = findJSON( - "watch.html", - "response", - body, - /\bytInitialData("\])?\s*=\s*\{/i, - "", - "{", - ); - info.html5player = getHTML5player(body); - return info; -}; - -const INFO_HOST = "www.youtube.com"; -const INFO_PATH = "/get_video_info"; -const VIDEO_EURL = "https://youtube.googleapis.com/v/"; -const getVideoInfoPage = async (id: string, options: GetInfoOptions) => { - const url = new URL(`https://${INFO_HOST}${INFO_PATH}`); - url.searchParams.set("video_id", id); - url.searchParams.set("eurl", VIDEO_EURL + id); - url.searchParams.set("ps", "default"); - url.searchParams.set("gl", "US"); - url.searchParams.set("hl", options.lang || "en"); - url.searchParams.set("c", "TVHTML5"); - url.searchParams.set("cver", `7${cver.substr(1)}`); - url.searchParams.set("html5", "1"); - let body = await request(url.toString(), options).then((e) => e.text()); - let info = querystring.decode(body); - info.player_response = findPlayerResponse("get_video_info", info); - return info; -}; - -/** - * @param {Object} player_response - * @returns {Array.} - */ -const parseFormats = (player_response: any) => { - let formats: any[] = []; - if (player_response && player_response.streamingData) { - formats = formats - .concat(player_response.streamingData.formats || []) - .concat(player_response.streamingData.adaptiveFormats || []); - } - return formats; -}; - -/** - * Gets info from a video additional formats and deciphered URLs. - */ -export const getInfo = async ( - id: string, - options: GetInfoOptions = {}, -): Promise => { - let info = await getBasicInfo(id, options); - const hasManifest = info.player_response && - info.player_response.streamingData && - (info.player_response.streamingData.dashManifestUrl || - info.player_response.streamingData.hlsManifestUrl); - let funcs = []; - if (info.formats.length) { - info.html5player = (info.html5player || - getHTML5player(await getWatchHTMLPageBody(id, options)) || - getHTML5player(await getEmbedPageBody(id, options)))!; - if (!info.html5player) { - throw Error("Unable to find html5player file"); - } - const html5player = new URL(info.html5player, BASE_URL).toString(); - funcs.push(sig.decipherFormats(info.formats, html5player, options)); - } - if (hasManifest && info.player_response.streamingData.dashManifestUrl) { - let url = info.player_response.streamingData.dashManifestUrl; - funcs.push(getDashManifest(url, options)); - } - if (hasManifest && info.player_response.streamingData.hlsManifestUrl) { - let url = info.player_response.streamingData.hlsManifestUrl; - funcs.push(getM3U8(url, options)); - } - - let results = await Promise.all(funcs); - info.formats = Object.values(Object.assign({}, ...results)); - info.formats = info.formats.map(formatUtils.addFormatMeta); - info.formats.sort(formatUtils.sortFormats); - info.full = true; - return info; -}; - -/** - * Gets additional DASH formats. - * - * @param {string} url - * @param {Object} options - * @returns {Promise>} - */ -const getDashManifest = (url: string, options: any) => - new Promise((resolve, reject) => { - let formats: any = {}; - const parser = new sax.SAXParser(false, {}); - parser.onerror = reject; - let adaptationSet: any; - parser.onopentag = (node: any) => { - if (node.name === "ADAPTATIONSET") { - adaptationSet = node.attributes; - } else if (node.name === "REPRESENTATION") { - const itag = parseInt(node.attributes.ID as any); - if (!isNaN(itag)) { - formats[url] = Object.assign( - { - itag, - url, - bitrate: parseInt(node.attributes.BANDWIDTH as any), - mimeType: - `${adaptationSet.MIMETYPE}; codecs="${node.attributes.CODECS}"`, - }, - node.attributes.HEIGHT - ? { - width: parseInt(node.attributes.WIDTH as any), - height: parseInt(node.attributes.HEIGHT as any), - fps: parseInt(node.attributes.FRAMERATE as any), - } - : { - audioSampleRate: node.attributes.AUDIOSAMPLINGRATE, - }, - ); - } - } - }; - parser.onend = () => { - resolve(formats); - }; - const req = request(new URL(url, BASE_URL).toString(), options); - - req - .then(async (res) => { - for await (const chunk of res.body!) { - parser.write(new TextDecoder().decode(chunk)); - } - parser.close.bind(parser)(); - }) - .catch(reject); - }); - -/** - * Gets additional formats. - * - * @param {string} url - * @param {Object} options - * @returns {Promise>} - */ -const getM3U8 = async (_url: string, options: any) => { - let url = new URL(_url, BASE_URL); - let body = await request(url.toString(), options.requestOptions).then((e) => - e.text() - ); - let formats: any = {}; - body - .split("\n") - .filter((line) => /^https?:\/\//.test(line)) - .forEach((line) => { - const itag = parseInt(line.match(/\/itag\/(\d+)\//)![1]); - formats[line] = { itag, url: line }; - }); - return formats; -}; +import { sax } from "../deps.ts"; +import { Cache } from "./cache.ts"; +import * as sig from "./sig.ts"; +import * as urlUtils from "./url_utils.ts"; +import * as utils from "./utils.ts"; +import * as formatUtils from "./format_util.ts"; +import * as extras from "./info_extras.ts"; +import { GetInfoOptions, VideoInfo } from "./types.ts"; +import { request } from "./request.ts"; + +let cver = "2.20210622.10.00"; + +const BASE_URL = "https://www.youtube.com/watch?v="; + +export const cache = new Cache(); +export const cookieCache = new Cache(1000 * 60 * 60 * 24); +export const watchPageCache = new Cache(); + +export class UnrecoverableError extends Error { + name = "UnrecoverableError"; +} + +export const USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.101 Safari/537.36"; + +const AGE_RESTRICTED_URLS = [ + "support.google.com/youtube/?p=age_restrictions", + "youtube.com/t/community_guidelines", +]; + +export async function getBasicInfo( + id: string, + options: GetInfoOptions = {}, +): Promise { + id = urlUtils.getVideoID(id); + options.headers = Object.assign( + {}, + { + // eslint-disable-next-line max-len + "User-Agent": USER_AGENT, + }, + options.headers, + ); + const validate = (info: any) => { + let playErr = utils.playError( + info.player_response, + ["ERROR"], + UnrecoverableError as any, + ); + let privateErr = privateVideoError(info.player_response); + if (playErr || privateErr) { + throw playErr || privateErr; + } + return ( + info && + info.player_response && + (info.player_response.streamingData || + isRental(info.player_response) || + isNotYetBroadcasted(info.player_response)) + ); + }; + let info = await pipeline([id, options], validate, {}, [ + getWatchHTMLPage, + getWatchJSONPage, + getVideoInfoPage, + ]); + + if (info.player_response === undefined) throw new Error("404 - Not found"); + + Object.assign(info, { + formats: parseFormats(info.player_response), + related_videos: extras.getRelatedVideos(info), + }); + + // Add additional properties to info. + const media = extras.getMedia(info); + let additional = { + author: extras.getAuthor(info), + media, + likes: extras.getLikes(info), + dislikes: extras.getDislikes(info), + age_restricted: !!( + media && + media.notice_url && + AGE_RESTRICTED_URLS.some((url) => media.notice_url.includes(url)) + ), + + // Give the standard link to the video. + video_url: BASE_URL + id, + storyboards: extras.getStoryboards(info), + }; + + info.videoDetails = extras.cleanVideoDetails( + Object.assign( + {}, + info.player_response && + info.player_response.microformat && + info.player_response.microformat.playerMicroformatRenderer, + info.player_response && info.player_response.videoDetails, + additional, + ), + info, + ); + + return info; +} + +const privateVideoError = (player_response: any) => { + let playability = player_response && player_response.playabilityStatus; + if ( + playability && + playability.status === "LOGIN_REQUIRED" && + playability.messages && + playability.messages.filter((m: any) => /This is a private video/.test(m)) + .length + ) { + return new UnrecoverableError( + playability.reason || (playability.messages && playability.messages[0]), + ); + } else { + return null; + } +}; + +const isRental = (player_response: any) => { + let playability = player_response.playabilityStatus; + return ( + playability && + playability.status === "UNPLAYABLE" && + playability.errorScreen && + playability.errorScreen.playerLegacyDesktopYpcOfferRenderer + ); +}; + +const isNotYetBroadcasted = (player_response: any) => { + let playability = player_response.playabilityStatus; + return playability && playability.status === "LIVE_STREAM_OFFLINE"; +}; + +const getWatchHTMLURL = (id: string, options: any) => + `${BASE_URL + id}&hl=${options.lang || "en"}`; +const getWatchHTMLPageBody = (id: string, options: any) => { + const url = getWatchHTMLURL(id, options); + return watchPageCache.getOrSet(url, () => + request(url, options) + .then((r) => r.text()) + .then((t) => { + return t; + })); +}; + +const EMBED_URL = "https://www.youtube.com/embed/"; +const getEmbedPageBody = (id: string, options: GetInfoOptions = {}) => { + const embedUrl = `${EMBED_URL + id}?hl=${options.lang || "en"}`; + return request(embedUrl, options).then((e) => e.text()); +}; + +const getHTML5player = (body: string) => { + let html5playerRes = + /|"jsUrl":"([^"]+)"/ + .exec( + body, + ); + return html5playerRes ? html5playerRes[1] || html5playerRes[2] : null; +}; + +const getIdentityToken = ( + id: string, + options: any, + key: string, + throwIfNotFound: boolean, +) => + cookieCache.getOrSet(key, async () => { + let page = await getWatchHTMLPageBody(id, options); + let match = page.match(/(["'])ID_TOKEN\1[:,]\s?"([^"]+)"/); + if (!match && throwIfNotFound) { + throw new UnrecoverableError( + "Cookie header used in request, but unable to find YouTube identity token", + ); + } + return match && match[2]; + }); + +/** + * Goes through each endpoint in the pipeline, retrying on failure if the error is recoverable. + * If unable to succeed with one endpoint, moves onto the next one. + */ +const pipeline = async ( + args: any[], + validate: CallableFunction, + retryOptions: any, + endpoints: CallableFunction[], +) => { + let info; + for (let func of endpoints) { + try { + const newInfo = await retryFunc(func, args.concat([info]), retryOptions); + if (newInfo && newInfo.player_response) { + newInfo.player_response.videoDetails = assign( + info && info.player_response && info.player_response.videoDetails, + newInfo.player_response.videoDetails, + ); + newInfo.player_response = assign( + info && info.player_response, + newInfo.player_response, + ); + } + info = assign(info, newInfo); + if (validate(info, false)) { + break; + } + } catch (err) { + if ( + err instanceof UnrecoverableError || + func === endpoints[endpoints.length - 1] + ) { + throw err; + } + // Unable to find video metadata... so try next endpoint. + } + } + return info; +}; + +/** + * Like Object.assign(), but ignores `null` and `undefined` from `source`. + * + * @param {Object} target + * @param {Object} source + * @returns {Object} + */ +const assign = (target: any, source: any) => { + if (!target || !source) { + return target || source; + } + for (let [key, value] of Object.entries(source)) { + if (value !== null && value !== undefined) { + target[key] = value; + } + } + return target; +}; + +/** + * Given a function, calls it with `args` until it's successful, + * or until it encounters an unrecoverable error. + * Currently, any error from miniget is considered unrecoverable. Errors such as + * too many redirects, invalid URL, status code 404, status code 502. + * + * @param {Function} func + * @param {Array.} args + * @param {Object} options + * @param {number} options.maxRetries + * @param {Object} options.backoff + * @param {number} options.backoff.inc + */ +const retryFunc = async (func: CallableFunction, args: any[], options: any) => { + let currentTry = 0, + result; + while (currentTry <= (options.maxRetries ?? 1)) { + try { + result = await func(...args); + break; + } catch (err) { + if ( + err instanceof UnrecoverableError || + err instanceof TypeError || + err.statusCode < 500 || + currentTry >= options.maxRetries + ) { + throw err; + } + let wait = Math.min( + ++currentTry * (options.backoff?.inc ?? 0), + options.backoff?.max ?? 0, + ); + await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + return result; +}; + +const jsonClosingChars = /^[)\]}'\s]+/; +const parseJSON = (source: any, varName: any, json: any) => { + if (!json || typeof json === "object") { + return json; + } else { + try { + json = json.replace(jsonClosingChars, ""); + return JSON.parse(json); + } catch (err) { + throw Error(`Error parsing ${varName} in ${source}: ${err.message}`); + } + } +}; + +const findJSON = ( + source: any, + varName: string, + body: any, + left: any, + right: any, + prependJSON: any, +) => { + let jsonStr = utils.between(body, left, right); + if (!jsonStr) { + throw Error(`Could not find ${varName} in ${source}`); + } + return parseJSON( + source, + varName, + utils.cutAfterJSON(`${prependJSON}${jsonStr}`), + ); +}; + +const findPlayerResponse = (source: any, info: any) => { + const player_response = info && + ((info.args && info.args.player_response) || + info.player_response || + info.playerResponse || + info.embedded_player_response); + return parseJSON(source, "player_response", player_response); +}; + +const getWatchJSONURL = (id: string, options: GetInfoOptions) => + `${getWatchHTMLURL(id, options)}&pbj=1`; +const getWatchJSONPage = async (id: string, options: GetInfoOptions) => { + const reqOptions = Object.assign({ headers: {} }, options); + let cookie = (reqOptions.headers as any).Cookie || + (reqOptions.headers as any).cookie; + reqOptions.headers = Object.assign( + { + "x-youtube-client-name": "1", + "x-youtube-client-version": cver, + "x-youtube-identity-token": cookieCache.get(cookie || "browser") || "", + }, + reqOptions.headers, + ); + + const setIdentityToken = async (key: string, throwIfNotFound: boolean) => { + if ((reqOptions.headers as any)["x-youtube-identity-token"]) { + return; + } + (reqOptions.headers as any)[ + "x-youtube-identity-token" + ] = await getIdentityToken(id, options, key, throwIfNotFound); + }; + + if (cookie) { + await setIdentityToken(cookie, true); + } + + const jsonUrl = getWatchJSONURL(id, options); + let body = await request(jsonUrl, reqOptions).then((e) => e.text()); + let parsedBody = parseJSON("watch.json", "body", body); + if (parsedBody.reload === "now") { + await setIdentityToken("browser", false); + } + if (parsedBody.reload === "now" || !Array.isArray(parsedBody)) { + throw Error("Unable to retrieve video metadata in watch.json"); + } + let info = parsedBody.reduce((part, curr) => Object.assign(curr, part), {}); + info.player_response = findPlayerResponse("watch.json", info); + info.html5player = info.player && info.player.assets && info.player.assets.js; + + return info; +}; + +const getWatchHTMLPage = async (id: string, options: GetInfoOptions) => { + let body = await getWatchHTMLPageBody(id, options); + let info: any = { page: "watch" }; + try { + cver = utils.between(body, '{"key":"cver","value":"', '"}'); + info.player_response = findJSON( + "watch.html", + "player_response", + body, + /\bytInitialPlayerResponse\s*=\s*\{/i, + "", + "{", + ); + } catch (err) { + let args = findJSON( + "watch.html", + "player_response", + body, + /\bytplayer\.config\s*=\s*{/, + "", + "{", + ); + info.player_response = findPlayerResponse("watch.html", args); + } + info.response = findJSON( + "watch.html", + "response", + body, + /\bytInitialData("\])?\s*=\s*\{/i, + "", + "{", + ); + info.html5player = getHTML5player(body); + return info; +}; + +const INFO_HOST = "www.youtube.com"; +const INFO_PATH = "/get_video_info"; +const VIDEO_EURL = "https://youtube.googleapis.com/v/"; +const getVideoInfoPage = async (id: string, options: GetInfoOptions) => { + const url = new URL(`https://${INFO_HOST}${INFO_PATH}`); + url.searchParams.set("video_id", id); + url.searchParams.set("eurl", VIDEO_EURL + id); + url.searchParams.set("ps", "default"); + url.searchParams.set("gl", "US"); + url.searchParams.set("hl", options.lang || "en"); + url.searchParams.set("c", "TVHTML5"); + url.searchParams.set("cver", `7${cver.substr(1)}`); + url.searchParams.set("html5", "1"); + let body = await request(url.toString(), options).then((e) => e.text()); + let info = Object.fromEntries([ ... new URLSearchParams(body).entries() ]); + info.player_response = findPlayerResponse("get_video_info", info); + return info; +}; + +/** + * @param {Object} player_response + * @returns {Array.} + */ +const parseFormats = (player_response: any) => { + let formats: any[] = []; + if (player_response && player_response.streamingData) { + formats = formats + .concat(player_response.streamingData.formats || []) + .concat(player_response.streamingData.adaptiveFormats || []); + } + return formats; +}; + +/** + * Gets info from a video additional formats and deciphered URLs. + */ +export const getInfo = async ( + id: string, + options: GetInfoOptions = {}, +): Promise => { + let info = await getBasicInfo(id, options); + const hasManifest = info.player_response && + info.player_response.streamingData && + (info.player_response.streamingData.dashManifestUrl || + info.player_response.streamingData.hlsManifestUrl); + let funcs = []; + if (info.formats.length) { + info.html5player = (info.html5player || + getHTML5player(await getWatchHTMLPageBody(id, options)) || + getHTML5player(await getEmbedPageBody(id, options)))!; + if (!info.html5player) { + throw Error("Unable to find html5player file"); + } + const html5player = new URL(info.html5player, BASE_URL).toString(); + funcs.push(sig.decipherFormats(info.formats, html5player, options)); + } + if (hasManifest && info.player_response.streamingData.dashManifestUrl) { + let url = info.player_response.streamingData.dashManifestUrl; + funcs.push(getDashManifest(url, options)); + } + if (hasManifest && info.player_response.streamingData.hlsManifestUrl) { + let url = info.player_response.streamingData.hlsManifestUrl; + funcs.push(getM3U8(url, options)); + } + + let results = await Promise.all(funcs); + info.formats = Object.values(Object.assign({}, ...results)); + info.formats = info.formats.map(formatUtils.addFormatMeta); + info.formats.sort(formatUtils.sortFormats); + info.full = true; + return info; +}; + +/** + * Gets additional DASH formats. + * + * @param {string} url + * @param {Object} options + * @returns {Promise>} + */ +const getDashManifest = (url: string, options: any) => + new Promise((resolve, reject) => { + let formats: any = {}; + const parser = new sax.SAXParser(false, {}); + parser.onerror = reject; + let adaptationSet: any; + parser.onopentag = (node: any) => { + if (node.name === "ADAPTATIONSET") { + adaptationSet = node.attributes; + } else if (node.name === "REPRESENTATION") { + const itag = parseInt(node.attributes.ID as any); + if (!isNaN(itag)) { + formats[url] = Object.assign( + { + itag, + url, + bitrate: parseInt(node.attributes.BANDWIDTH as any), + mimeType: + `${adaptationSet.MIMETYPE}; codecs="${node.attributes.CODECS}"`, + }, + node.attributes.HEIGHT + ? { + width: parseInt(node.attributes.WIDTH as any), + height: parseInt(node.attributes.HEIGHT as any), + fps: parseInt(node.attributes.FRAMERATE as any), + } + : { + audioSampleRate: node.attributes.AUDIOSAMPLINGRATE, + }, + ); + } + } + }; + parser.onend = () => { + resolve(formats); + }; + const req = request(new URL(url, BASE_URL).toString(), options); + + req + .then(async (res) => { + for await (const chunk of res.body!) { + parser.write(new TextDecoder().decode(chunk)); + } + parser.close.bind(parser)(); + }) + .catch(reject); + }); + +/** + * Gets additional formats. + * + * @param {string} url + * @param {Object} options + * @returns {Promise>} + */ +const getM3U8 = async (_url: string, options: any) => { + let url = new URL(_url, BASE_URL); + let body = await request(url.toString(), options.requestOptions).then((e) => + e.text() + ); + let formats: any = {}; + body + .split("\n") + .filter((line) => /^https?:\/\//.test(line)) + .forEach((line) => { + const itag = parseInt(line.match(/\/itag\/(\d+)\//)![1]); + formats[line] = { itag, url: line }; + }); + return formats; +}; diff --git a/src/info_extras.ts b/src/info_extras.ts index 7d39a1f..fecbeea 100644 --- a/src/info_extras.ts +++ b/src/info_extras.ts @@ -1,367 +1,369 @@ -import { parseTimestamp, querystring as qs } from "../deps.ts"; -import * as utils from "./utils.ts"; - -const BASE_URL = "https://www.youtube.com/watch?v="; -const TITLE_TO_CATEGORY = { - song: { name: "Music", url: "https://music.youtube.com/" }, -}; - -const getText = (obj: any) => - obj ? (obj.runs ? obj.runs[0].text : obj.simpleText) : null; - -export const getMedia = (info: any) => { - let media: any = {}; - let results: any[] = []; - try { - results = - info.response.contents.twoColumnWatchNextResults.results.results.contents; - } catch (err) { - // Do nothing - } - - let result = results.find((v: any) => v.videoSecondaryInfoRenderer); - if (!result) { - return {}; - } - - try { - let metadataRows = ( - result.metadataRowContainer || - result.videoSecondaryInfoRenderer.metadataRowContainer - ).metadataRowContainerRenderer.rows; - for (let row of metadataRows) { - if (row.metadataRowRenderer) { - let title = getText(row.metadataRowRenderer.title).toLowerCase(); - let contents = row.metadataRowRenderer.contents[0]; - media[title] = getText(contents); - let runs = contents.runs; - if (runs && runs[0].navigationEndpoint) { - media[`${title}_url`] = new URL( - runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url, - BASE_URL, - ).toString(); - } - if (title in TITLE_TO_CATEGORY) { - media.category = (TITLE_TO_CATEGORY as any)[title].name; - media.category_url = (TITLE_TO_CATEGORY as any)[title].url; - } - } else if (row.richMetadataRowRenderer) { - let contents = row.richMetadataRowRenderer.contents; - let boxArt = contents.filter( - (meta: any) => - meta.richMetadataRenderer.style === - "RICH_METADATA_RENDERER_STYLE_BOX_ART", - ); - for (let { richMetadataRenderer } of boxArt) { - let meta = richMetadataRenderer; - media.year = getText(meta.subtitle); - let type = getText(meta.callToAction).split(" ")[1]; - media[type] = getText(meta.title); - media[`${type}_url`] = new URL( - meta.endpoint.commandMetadata.webCommandMetadata.url, - BASE_URL, - ).toString(); - media.thumbnails = meta.thumbnail.thumbnails; - } - let topic = contents.filter( - (meta: any) => - meta.richMetadataRenderer.style === - "RICH_METADATA_RENDERER_STYLE_TOPIC", - ); - for (let { richMetadataRenderer } of topic) { - let meta = richMetadataRenderer; - media.category = getText(meta.title); - media.category_url = new URL( - meta.endpoint.commandMetadata.webCommandMetadata.url, - BASE_URL, - ).toString(); - } - } - } - } catch (err) { - // Do nothing. - } - - return media; -}; - -const isVerified = (badges: any[]) => - !!( - badges && badges.find((b) => b.metadataBadgeRenderer.tooltip === "Verified") - ); - -export const getAuthor = (info: any) => { - let channelId, - thumbnails = [], - subscriberCount, - verified = false; - try { - let results = - info.response.contents.twoColumnWatchNextResults.results.results.contents; - let v = results.find( - (v2: any) => - v2.videoSecondaryInfoRenderer && - v2.videoSecondaryInfoRenderer.owner && - v2.videoSecondaryInfoRenderer.owner.videoOwnerRenderer, - ); - let videoOwnerRenderer = - v.videoSecondaryInfoRenderer.owner.videoOwnerRenderer; - channelId = videoOwnerRenderer.navigationEndpoint.browseEndpoint.browseId; - thumbnails = videoOwnerRenderer.thumbnail.thumbnails.map( - (thumbnail: any) => { - thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); - return thumbnail; - }, - ); - subscriberCount = utils.parseAbbreviatedNumber( - getText(videoOwnerRenderer.subscriberCountText), - ); - verified = isVerified(videoOwnerRenderer.badges); - } catch (err) { - // Do nothing. - } - try { - let videoDetails = info.player_response.microformat && - info.player_response.microformat.playerMicroformatRenderer; - let id = (videoDetails && videoDetails.channelId) || - channelId || - info.player_response.videoDetails.channelId; - let author = { - id: id, - name: videoDetails - ? videoDetails.ownerChannelName - : info.player_response.videoDetails.author, - user: videoDetails - ? videoDetails.ownerProfileUrl.split("/").slice(-1)[0] - : null, - channel_url: `https://www.youtube.com/channel/${id}`, - external_channel_url: videoDetails - ? `https://www.youtube.com/channel/${videoDetails.externalChannelId}` - : "", - user_url: videoDetails - ? new URL(videoDetails.ownerProfileUrl, BASE_URL).toString() - : "", - thumbnails, - verified, - subscriber_count: subscriberCount, - }; - return author; - } catch (err) { - return {}; - } -}; - -const parseRelatedVideo = (details: any, rvsParams: any) => { - if (!details) return; - try { - let viewCount = getText(details.viewCountText); - let shortViewCount = getText(details.shortViewCountText); - let rvsDetails = rvsParams.find((elem: any) => elem.id === details.videoId); - if (!/^\d/.test(shortViewCount)) { - shortViewCount = (rvsDetails && rvsDetails.short_view_count_text) || ""; - } - viewCount = (/^\d/.test(viewCount) ? viewCount : shortViewCount).split( - " ", - )[0]; - let browseEndpoint = - details.shortBylineText.runs[0].navigationEndpoint.browseEndpoint; - let channelId = browseEndpoint.browseId; - let name = getText(details.shortBylineText); - let user = (browseEndpoint.canonicalBaseUrl || "").split("/").slice(-1)[0]; - let video = { - id: details.videoId, - title: getText(details.title), - published: getText(details.publishedTimeText), - author: { - id: channelId, - name, - user, - channel_url: `https://www.youtube.com/channel/${channelId}`, - user_url: `https://www.youtube.com/user/${user}`, - thumbnails: details.channelThumbnail.thumbnails.map( - (thumbnail: any) => { - thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); - return thumbnail; - }, - ), - verified: isVerified(details.ownerBadges), - - [Symbol.toPrimitive]() { - console.warn( - `\`relatedVideo.author\` will be removed in a near future release, ` + - `use \`relatedVideo.author.name\` instead.`, - ); - return video.author.name; - }, - }, - short_view_count_text: shortViewCount.split(" ")[0], - view_count: viewCount.replace(/,/g, ""), - length_seconds: details.lengthText - ? Math.floor(parseTimestamp(getText(details.lengthText)) / 1000) - : rvsParams && `${rvsParams.length_seconds}`, - thumbnails: details.thumbnail.thumbnails, - richThumbnails: details.richThumbnail - ? details.richThumbnail.movingThumbnailRenderer.movingThumbnailDetails - .thumbnails - : [], - isLive: !!( - details.badges && - details.badges.find( - (b: any) => b.metadataBadgeRenderer.label === "LIVE NOW", - ) - ), - }; - return video; - } catch (err) { - // Skip. - } -}; - -export const getRelatedVideos = (info: any) => { - let rvsParams = [], - secondaryResults = []; - try { - rvsParams = info.response.webWatchNextResponseExtensionData.relatedVideoArgs - .split(",") - .map((e: any) => qs.parse(e)); - } catch (err) { - // Do nothing. - } - try { - secondaryResults = - info.response.contents.twoColumnWatchNextResults.secondaryResults - .secondaryResults.results; - } catch (err) { - return []; - } - let videos = []; - for (let result of secondaryResults || []) { - let details = result.compactVideoRenderer; - if (details) { - let video = parseRelatedVideo(details, rvsParams); - if (video) videos.push(video); - } else { - let autoplay = result.compactAutoplayRenderer || - result.itemSectionRenderer; - if (!autoplay || !Array.isArray(autoplay.contents)) continue; - for (let content of autoplay.contents) { - let video = parseRelatedVideo(content.compactVideoRenderer, rvsParams); - if (video) videos.push(video); - } - } - } - return videos; -}; - -/** - * Get like count. - */ -export const getLikes = (info: any) => { - try { - let contents = - info.response.contents.twoColumnWatchNextResults.results.results.contents; - let video = contents.find((r: any) => r.videoPrimaryInfoRenderer); - let buttons = - video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons; - let like = buttons.find( - (b: any) => - b.toggleButtonRenderer && - b.toggleButtonRenderer.defaultIcon.iconType === "LIKE", - ); - return parseInt( - like.toggleButtonRenderer.defaultText.accessibility.accessibilityData - .label.replace( - /\D+/g, - "", - ), - ); - } catch (err) { - return null; - } -}; - -export const getDislikes = (info: any) => { - try { - let contents = - info.response.contents.twoColumnWatchNextResults.results.results.contents; - let video = contents.find((r: any) => r.videoPrimaryInfoRenderer); - let buttons = - video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons; - let dislike = buttons.find( - (b: any) => - b.toggleButtonRenderer && - b.toggleButtonRenderer.defaultIcon.iconType === "DISLIKE", - ); - return parseInt( - dislike.toggleButtonRenderer.defaultText.accessibility.accessibilityData - .label.replace( - /\D+/g, - "", - ), - ); - } catch (err) { - return null; - } -}; - -export const cleanVideoDetails = (videoDetails: any, info: any) => { - videoDetails.thumbnails = videoDetails.thumbnail.thumbnails; - delete videoDetails.thumbnail; - videoDetails.description = videoDetails.shortDescription || - getText(videoDetails.description); - delete videoDetails.shortDescription; - - // Use more reliable `lengthSeconds` from `playerMicroformatRenderer`. - videoDetails.lengthSeconds = (info.player_response.microformat && - info.player_response.microformat.playerMicroformatRenderer.lengthSeconds) || - info.player_response.videoDetails.lengthSeconds; - return videoDetails; -}; - -export const getStoryboards = (info: any) => { - const parts = info.player_response?.storyboards && - info.player_response.storyboards.playerStoryboardSpecRenderer && - info.player_response.storyboards.playerStoryboardSpecRenderer.spec && - info.player_response.storyboards.playerStoryboardSpecRenderer.spec.split( - "|", - ); - - if (!parts) return []; - - const url = new URL(parts.shift()); - - return parts.map((part: any, i: number) => { - let [ - thumbnailWidth, - thumbnailHeight, - thumbnailCount, - columns, - rows, - interval, - nameReplacement, - sigh, - ] = part.split("#"); - - url.searchParams.set("sigh", sigh); - - thumbnailCount = parseInt(thumbnailCount, 10); - columns = parseInt(columns, 10); - rows = parseInt(rows, 10); - - const storyboardCount = Math.ceil(thumbnailCount / (columns * rows)); - - return { - templateUrl: url - .toString() - .replace("$L", i.toString()) - .replace("$N", nameReplacement), - thumbnailWidth: parseInt(thumbnailWidth, 10), - thumbnailHeight: parseInt(thumbnailHeight, 10), - thumbnailCount, - interval: parseInt(interval, 10), - columns, - rows, - storyboardCount, - }; - }); -}; +import { parseTimestamp } from "../deps.ts"; +import * as utils from "./utils.ts"; + +const BASE_URL = "https://www.youtube.com/watch?v="; +const TITLE_TO_CATEGORY = { + song: { name: "Music", url: "https://music.youtube.com/" }, +}; + +const getText = (obj: any) => + obj ? (obj.runs ? obj.runs[0].text : obj.simpleText) : null; + +export const getMedia = (info: any) => { + let media: any = {}; + let results: any[] = []; + try { + results = + info.response.contents.twoColumnWatchNextResults.results.results.contents; + } catch (err) { + // Do nothing + } + + let result = results.find((v: any) => v.videoSecondaryInfoRenderer); + if (!result) { + return {}; + } + + try { + let metadataRows = ( + result.metadataRowContainer || + result.videoSecondaryInfoRenderer.metadataRowContainer + ).metadataRowContainerRenderer.rows; + for (let row of metadataRows) { + if (row.metadataRowRenderer) { + let title = getText(row.metadataRowRenderer.title).toLowerCase(); + let contents = row.metadataRowRenderer.contents[0]; + media[title] = getText(contents); + let runs = contents.runs; + if (runs && runs[0].navigationEndpoint) { + media[`${title}_url`] = new URL( + runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url, + BASE_URL, + ).toString(); + } + if (title in TITLE_TO_CATEGORY) { + media.category = (TITLE_TO_CATEGORY as any)[title].name; + media.category_url = (TITLE_TO_CATEGORY as any)[title].url; + } + } else if (row.richMetadataRowRenderer) { + let contents = row.richMetadataRowRenderer.contents; + let boxArt = contents.filter( + (meta: any) => + meta.richMetadataRenderer.style === + "RICH_METADATA_RENDERER_STYLE_BOX_ART", + ); + for (let { richMetadataRenderer } of boxArt) { + let meta = richMetadataRenderer; + media.year = getText(meta.subtitle); + let type = getText(meta.callToAction).split(" ")[1]; + media[type] = getText(meta.title); + media[`${type}_url`] = new URL( + meta.endpoint.commandMetadata.webCommandMetadata.url, + BASE_URL, + ).toString(); + media.thumbnails = meta.thumbnail.thumbnails; + } + let topic = contents.filter( + (meta: any) => + meta.richMetadataRenderer.style === + "RICH_METADATA_RENDERER_STYLE_TOPIC", + ); + for (let { richMetadataRenderer } of topic) { + let meta = richMetadataRenderer; + media.category = getText(meta.title); + media.category_url = new URL( + meta.endpoint.commandMetadata.webCommandMetadata.url, + BASE_URL, + ).toString(); + } + } + } + } catch (err) { + // Do nothing. + } + + return media; +}; + +const isVerified = (badges: any[]) => + !!( + badges && badges.find((b) => b.metadataBadgeRenderer.tooltip === "Verified") + ); + +export const getAuthor = (info: any) => { + let channelId, + thumbnails = [], + subscriberCount, + verified = false; + try { + let results = + info.response.contents.twoColumnWatchNextResults.results.results.contents; + let v = results.find( + (v2: any) => + v2.videoSecondaryInfoRenderer && + v2.videoSecondaryInfoRenderer.owner && + v2.videoSecondaryInfoRenderer.owner.videoOwnerRenderer, + ); + let videoOwnerRenderer = + v.videoSecondaryInfoRenderer.owner.videoOwnerRenderer; + channelId = videoOwnerRenderer.navigationEndpoint.browseEndpoint.browseId; + thumbnails = videoOwnerRenderer.thumbnail.thumbnails.map( + (thumbnail: any) => { + thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); + return thumbnail; + }, + ); + subscriberCount = utils.parseAbbreviatedNumber( + getText(videoOwnerRenderer.subscriberCountText), + ); + verified = isVerified(videoOwnerRenderer.badges); + } catch (err) { + // Do nothing. + } + try { + let videoDetails = info.player_response.microformat && + info.player_response.microformat.playerMicroformatRenderer; + let id = (videoDetails && videoDetails.channelId) || + channelId || + info.player_response.videoDetails.channelId; + let author = { + id: id, + name: videoDetails + ? videoDetails.ownerChannelName + : info.player_response.videoDetails.author, + user: videoDetails + ? videoDetails.ownerProfileUrl.split("/").slice(-1)[0] + : null, + channel_url: `https://www.youtube.com/channel/${id}`, + external_channel_url: videoDetails + ? `https://www.youtube.com/channel/${videoDetails.externalChannelId}` + : "", + user_url: videoDetails + ? new URL(videoDetails.ownerProfileUrl, BASE_URL).toString() + : "", + thumbnails, + verified, + subscriber_count: subscriberCount, + }; + return author; + } catch (err) { + return {}; + } +}; + +const parseRelatedVideo = (details: any, rvsParams: any) => { + if (!details) return; + try { + let viewCount = getText(details.viewCountText); + let shortViewCount = getText(details.shortViewCountText); + let rvsDetails = rvsParams.find((elem: any) => elem.id === details.videoId); + if (!/^\d/.test(shortViewCount)) { + shortViewCount = (rvsDetails && rvsDetails.short_view_count_text) || ""; + } + viewCount = (/^\d/.test(viewCount) ? viewCount : shortViewCount).split( + " ", + )[0]; + let browseEndpoint = + details.shortBylineText.runs[0].navigationEndpoint.browseEndpoint; + let channelId = browseEndpoint.browseId; + let name = getText(details.shortBylineText); + let user = (browseEndpoint.canonicalBaseUrl || "").split("/").slice(-1)[0]; + let video = { + id: details.videoId, + title: getText(details.title), + published: getText(details.publishedTimeText), + author: { + id: channelId, + name, + user, + channel_url: `https://www.youtube.com/channel/${channelId}`, + user_url: `https://www.youtube.com/user/${user}`, + thumbnails: details.channelThumbnail.thumbnails.map( + (thumbnail: any) => { + thumbnail.url = new URL(thumbnail.url, BASE_URL).toString(); + return thumbnail; + }, + ), + verified: isVerified(details.ownerBadges), + + [Symbol.toPrimitive]() { + console.warn( + `\`relatedVideo.author\` will be removed in a near future release, ` + + `use \`relatedVideo.author.name\` instead.`, + ); + return video.author.name; + }, + }, + short_view_count_text: shortViewCount.split(" ")[0], + view_count: viewCount.replace(/,/g, ""), + length_seconds: details.lengthText + ? Math.floor(parseTimestamp(getText(details.lengthText)) / 1000) + : rvsParams && `${rvsParams.length_seconds}`, + thumbnails: details.thumbnail.thumbnails, + richThumbnails: details.richThumbnail + ? details.richThumbnail.movingThumbnailRenderer.movingThumbnailDetails + .thumbnails + : [], + isLive: !!( + details.badges && + details.badges.find( + (b: any) => b.metadataBadgeRenderer.label === "LIVE NOW", + ) + ), + }; + return video; + } catch (err) { + // Skip. + } +}; + +export const getRelatedVideos = (info: any) => { + let rvsParams = [], + secondaryResults = []; + try { + rvsParams = info.response.webWatchNextResponseExtensionData.relatedVideoArgs + .split(",") + .map((e: any) => new URLSearchParams(e)) + .map(( params ) => [ ... params.entries() ] ) + .map(( params ) => Object.fromEntries(params) ); + } catch (err) { + // Do nothing. + } + try { + secondaryResults = + info.response.contents.twoColumnWatchNextResults.secondaryResults + .secondaryResults.results; + } catch (err) { + return []; + } + let videos = []; + for (let result of secondaryResults || []) { + let details = result.compactVideoRenderer; + if (details) { + let video = parseRelatedVideo(details, rvsParams); + if (video) videos.push(video); + } else { + let autoplay = result.compactAutoplayRenderer || + result.itemSectionRenderer; + if (!autoplay || !Array.isArray(autoplay.contents)) continue; + for (let content of autoplay.contents) { + let video = parseRelatedVideo(content.compactVideoRenderer, rvsParams); + if (video) videos.push(video); + } + } + } + return videos; +}; + +/** + * Get like count. + */ +export const getLikes = (info: any) => { + try { + let contents = + info.response.contents.twoColumnWatchNextResults.results.results.contents; + let video = contents.find((r: any) => r.videoPrimaryInfoRenderer); + let buttons = + video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons; + let like = buttons.find( + (b: any) => + b.toggleButtonRenderer && + b.toggleButtonRenderer.defaultIcon.iconType === "LIKE", + ); + return parseInt( + like.toggleButtonRenderer.defaultText.accessibility.accessibilityData + .label.replace( + /\D+/g, + "", + ), + ); + } catch (err) { + return null; + } +}; + +export const getDislikes = (info: any) => { + try { + let contents = + info.response.contents.twoColumnWatchNextResults.results.results.contents; + let video = contents.find((r: any) => r.videoPrimaryInfoRenderer); + let buttons = + video.videoPrimaryInfoRenderer.videoActions.menuRenderer.topLevelButtons; + let dislike = buttons.find( + (b: any) => + b.toggleButtonRenderer && + b.toggleButtonRenderer.defaultIcon.iconType === "DISLIKE", + ); + return parseInt( + dislike.toggleButtonRenderer.defaultText.accessibility.accessibilityData + .label.replace( + /\D+/g, + "", + ), + ); + } catch (err) { + return null; + } +}; + +export const cleanVideoDetails = (videoDetails: any, info: any) => { + videoDetails.thumbnails = videoDetails.thumbnail.thumbnails; + delete videoDetails.thumbnail; + videoDetails.description = videoDetails.shortDescription || + getText(videoDetails.description); + delete videoDetails.shortDescription; + + // Use more reliable `lengthSeconds` from `playerMicroformatRenderer`. + videoDetails.lengthSeconds = (info.player_response.microformat && + info.player_response.microformat.playerMicroformatRenderer.lengthSeconds) || + info.player_response.videoDetails.lengthSeconds; + return videoDetails; +}; + +export const getStoryboards = (info: any) => { + const parts = info.player_response?.storyboards && + info.player_response.storyboards.playerStoryboardSpecRenderer && + info.player_response.storyboards.playerStoryboardSpecRenderer.spec && + info.player_response.storyboards.playerStoryboardSpecRenderer.spec.split( + "|", + ); + + if (!parts) return []; + + const url = new URL(parts.shift()); + + return parts.map((part: any, i: number) => { + let [ + thumbnailWidth, + thumbnailHeight, + thumbnailCount, + columns, + rows, + interval, + nameReplacement, + sigh, + ] = part.split("#"); + + url.searchParams.set("sigh", sigh); + + thumbnailCount = parseInt(thumbnailCount, 10); + columns = parseInt(columns, 10); + rows = parseInt(rows, 10); + + const storyboardCount = Math.ceil(thumbnailCount / (columns * rows)); + + return { + templateUrl: url + .toString() + .replace("$L", i.toString()) + .replace("$N", nameReplacement), + thumbnailWidth: parseInt(thumbnailWidth, 10), + thumbnailHeight: parseInt(thumbnailHeight, 10), + thumbnailCount, + interval: parseInt(interval, 10), + columns, + rows, + storyboardCount, + }; + }); +}; diff --git a/src/request.ts b/src/request.ts index be415f2..d38df27 100644 --- a/src/request.ts +++ b/src/request.ts @@ -1,13 +1,13 @@ -export async function request( - url: string, - options?: RequestInit, -): Promise { - const res = await fetch(url, options); - if (res.status >= 400 && res.status < 600) { - await res.arrayBuffer(); // use the body to prevent leak - throw new Error( - `Request to ${url} Failed: ${res.status} ${res.statusText}`, - ); - } - return res; -} +export async function request( + url: string, + options?: RequestInit, +): Promise { + const res = await fetch(url, options); + if (res.status >= 400 && res.status < 600) { + await res.arrayBuffer(); // use the body to prevent leak + throw new Error( + `Request to ${url} Failed: ${res.status} ${res.statusText}`, + ); + } + return res; +} diff --git a/src/sig.ts b/src/sig.ts index 955b4bb..6ba0203 100644 --- a/src/sig.ts +++ b/src/sig.ts @@ -1,154 +1,153 @@ -import { querystring } from "../deps.ts"; -import { Cache } from "./cache.ts"; -import { between, cutAfterJSON } from "./utils.ts"; -import { request } from "./request.ts"; - -// A shared cache to keep track of html5player js functions. -export const cache = new Cache(); - -/** - * Extract signature deciphering and n parameter transform functions from html5player file. - * - * @param {string} html5playerfile - * @param {Object} options - * @returns {Promise>} - */ -export function getFunctions( - html5playerfile: string, - options: RequestInit -): string[] { - return cache.getOrSet(html5playerfile, async () => { - const res = await request(html5playerfile, options); - const body = await res.text(); - const functions = extractFunctions(body); - if (!functions || !functions.length) { - throw Error("Could not extract functions"); - } - cache.set(html5playerfile, functions); - return functions; - }); -} - -/** - * Extracts the actions that should be taken to decipher a signature - * and tranform the n parameter - * - * @param {string} body - * @returns {Array.} - */ -export function extractFunctions(body: string) { - const functions: string[] = []; - const extractManipulations = (caller: string) => { - const functionName = between(caller, `a=a.split("");`, `.`); - if (!functionName) return ""; - const functionStart = `var ${functionName}={`; - const ndx = body.indexOf(functionStart); - if (ndx < 0) return ""; - const subBody = body.slice(ndx + functionStart.length - 1); - return `var ${functionName}=${cutAfterJSON(subBody)}`; - }; - const extractDecipher = () => { - const functionName = between( - body, - `a.set("alr","yes");c&&(c=`, - `(decodeURIC` - ); - if (functionName && functionName.length) { - const functionStart = `${functionName}=function(a)`; - const ndx = body.indexOf(functionStart); - if (ndx >= 0) { - const subBody = body.slice(ndx + functionStart.length); - let functionBody = `var ${functionStart}${cutAfterJSON(subBody)}`; - functionBody = `${extractManipulations( - functionBody - )};${functionBody};${functionName}(sig);`; - functions.push(functionBody); - } - } - }; - const extractNCode = () => { - let functionName = between(body, `&&(b=a.get("n"))&&(b=`, `(b)`); - if (functionName.includes("[")) - functionName = between(body, `${functionName.split("[")[0]}=[`, `]`); - if (functionName && functionName.length) { - const functionStart = `${functionName}=function(a)`; - const ndx = body.indexOf(functionStart); - if (ndx >= 0) { - const end = body.indexOf('.join("")};', ndx); - const subBody = body.slice(ndx, end); - - const functionBody = `${subBody}.join("")};${functionName}(ncode);`; - functions.push(functionBody); - } - } - }; - extractDecipher(); - extractNCode(); - return functions; -} - -/** - * Apply decipher and n-transform to individual format - * - * @param {Object} format - * @param {vm.Script} decipherScript - * @param {vm.Script} nTransformScript - */ -export function setDownloadURL( - format: any, - decipherScript: ((sig: string) => string) | undefined, - nTransformScript: ((ncode: string) => string) | undefined -) { - const decipher = (url: string) => { - const args = querystring.parse(url) as any; - if (!args.s || !decipherScript) return args.url; - const components = new URL(decodeURIComponent(args.url)); - components.searchParams.set( - args.sp ? args.sp : "signature", - decipherScript(decodeURIComponent(args.s)) - ); - return components.toString(); - }; - const ncode = (url: string) => { - const components = new URL(decodeURIComponent(url)); - const n = components.searchParams.get("n"); - if (!n || !nTransformScript) return url; - components.searchParams.set("n", nTransformScript(n)); - return components.toString(); - }; - const cipher = !format.url; - const url = format.url || format.signatureCipher || format.cipher; - format.url = cipher ? ncode(decipher(url)) : ncode(url); - delete format.signatureCipher; - delete format.cipher; -} - -/** - * Applies decipher and n parameter transforms to all format URL's. - * - * @param {Array.} formats - * @param {string} html5player - * @param {Object} options - */ -export async function decipherFormats( - formats: any[], - html5player: string, - options: any -) { - const decipheredFormats: any = {}; - const functions = await getFunctions(html5player, options); - const decipherScript = functions.length - ? createFunc("sig")(functions[0]) - : undefined; - const nTransformScript = - functions.length > 1 ? createFunc("ncode")(functions[1]) : undefined; - formats.forEach((format) => { - setDownloadURL(format, decipherScript as any, nTransformScript as any); - decipheredFormats[format.url] = format; - }); - return decipheredFormats; -} - -function createFunc(param: string) { - return new Function("source", param, `return (${param}) => eval(source)`); -} +import { Cache } from "./cache.ts"; +import { between, cutAfterJSON } from "./utils.ts"; +import { request } from "./request.ts"; + +// A shared cache to keep track of html5player js functions. +export const cache = new Cache(); + +/** + * Extract signature deciphering and n parameter transform functions from html5player file. + * + * @param {string} html5playerfile + * @param {Object} options + * @returns {Promise>} + */ +export function getFunctions( + html5playerfile: string, + options: RequestInit +): string[] { + return cache.getOrSet(html5playerfile, async () => { + const res = await request(html5playerfile, options); + const body = await res.text(); + const functions = extractFunctions(body); + if (!functions || !functions.length) { + throw Error("Could not extract functions"); + } + cache.set(html5playerfile, functions); + return functions; + }); +} + +/** + * Extracts the actions that should be taken to decipher a signature + * and tranform the n parameter + * + * @param {string} body + * @returns {Array.} + */ +export function extractFunctions(body: string) { + const functions: string[] = []; + const extractManipulations = (caller: string) => { + const functionName = between(caller, `a=a.split("");`, `.`); + if (!functionName) return ""; + const functionStart = `var ${functionName}={`; + const ndx = body.indexOf(functionStart); + if (ndx < 0) return ""; + const subBody = body.slice(ndx + functionStart.length - 1); + return `var ${functionName}=${cutAfterJSON(subBody)}`; + }; + const extractDecipher = () => { + const functionName = between( + body, + `a.set("alr","yes");c&&(c=`, + `(decodeURIC` + ); + if (functionName && functionName.length) { + const functionStart = `${functionName}=function(a)`; + const ndx = body.indexOf(functionStart); + if (ndx >= 0) { + const subBody = body.slice(ndx + functionStart.length); + let functionBody = `var ${functionStart}${cutAfterJSON(subBody)}`; + functionBody = `${extractManipulations( + functionBody + )};${functionBody};${functionName}(sig);`; + functions.push(functionBody); + } + } + }; + const extractNCode = () => { + let functionName = between(body, `&&(b=a.get("n"))&&(b=`, `(b)`); + if (functionName.includes("[")) + functionName = between(body, `${functionName.split("[")[0]}=[`, `]`); + if (functionName && functionName.length) { + const functionStart = `${functionName}=function(a)`; + const ndx = body.indexOf(functionStart); + if (ndx >= 0) { + const end = body.indexOf('.join("")};', ndx); + const subBody = body.slice(ndx, end); + + const functionBody = `${subBody}.join("")};${functionName}(ncode);`; + functions.push(functionBody); + } + } + }; + extractDecipher(); + extractNCode(); + return functions; +} + +/** + * Apply decipher and n-transform to individual format + * + * @param {Object} format + * @param {vm.Script} decipherScript + * @param {vm.Script} nTransformScript + */ +export function setDownloadURL( + format: any, + decipherScript: ((sig: string) => string) | undefined, + nTransformScript: ((ncode: string) => string) | undefined +) { + const decipher = (url: string) => { + const args = new URLSearchParams(url) as any; + if (!args.has('s') || !decipherScript) return args.get('url'); + const components = new URL(decodeURIComponent(args.get('url'))); + components.searchParams.set( + args.has('sp') ? args.get('sp') : "signature", + decipherScript(decodeURIComponent(args.get('s'))) + ); + return components.toString(); + }; + const ncode = (url: string) => { + const components = new URL(decodeURIComponent(url)); + const n = components.searchParams.get("n"); + if (!n || !nTransformScript) return url; + components.searchParams.set("n", nTransformScript(n)); + return components.toString(); + }; + const cipher = !format.url; + const url = format.url || format.signatureCipher || format.cipher; + format.url = cipher ? ncode(decipher(url)) : ncode(url); + delete format.signatureCipher; + delete format.cipher; +} + +/** + * Applies decipher and n parameter transforms to all format URL's. + * + * @param {Array.} formats + * @param {string} html5player + * @param {Object} options + */ +export async function decipherFormats( + formats: any[], + html5player: string, + options: any +) { + const decipheredFormats: any = {}; + const functions = await getFunctions(html5player, options); + const decipherScript = functions.length + ? createFunc("sig")(functions[0]) + : undefined; + const nTransformScript = + functions.length > 1 ? createFunc("ncode")(functions[1]) : undefined; + formats.forEach((format) => { + setDownloadURL(format, decipherScript as any, nTransformScript as any); + decipheredFormats[format.url] = format; + }); + return decipheredFormats; +} + +function createFunc(param: string) { + return new Function("source", param, `return (${param}) => eval(source)`); +} diff --git a/src/stream.ts b/src/stream.ts index 8f97cbb..4bf05b1 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -1,8 +1,8 @@ -import { VideoFormat, VideoInfo } from "./types.ts"; - -export class VideoStream extends ReadableStream { - info!: VideoInfo; - format!: VideoFormat; - downloaded = 0; - total = 0; -} +import { VideoFormat, VideoInfo } from "./types.ts"; + +export class VideoStream extends ReadableStream { + info!: VideoInfo; + format!: VideoFormat; + downloaded = 0; + total = 0; +} diff --git a/src/types.ts b/src/types.ts index 5952804..dfe0a54 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,640 +1,640 @@ -export interface VideoFormat { - itag: number; - url: string; - mimeType?: string; - bitrate?: number; - audioBitrate?: number; - width?: number; - height?: number; - initRange?: { start: string; end: string }; - indexRange?: { start: string; end: string }; - lastModified: string; - contentLength: string; - quality: - | "tiny" - | "small" - | "medium" - | "large" - | "hd720" - | "hd1080" - | "hd1440" - | "hd2160" - | "highres" - | string; - qualityLabel: - | "144p" - | "144p 15fps" - | "144p60 HDR" - | "240p" - | "240p60 HDR" - | "270p" - | "360p" - | "360p60 HDR" - | "480p" - | "480p60 HDR" - | "720p" - | "720p60" - | "720p60 HDR" - | "1080p" - | "1080p60" - | "1080p60 HDR" - | "1440p" - | "1440p60" - | "1440p60 HDR" - | "2160p" - | "2160p60" - | "2160p60 HDR" - | "4320p" - | "4320p60"; - projectionType?: "RECTANGULAR"; - fps?: number; - averageBitrate?: number; - audioQuality?: "AUDIO_QUALITY_LOW" | "AUDIO_QUALITY_MEDIUM"; - colorInfo?: { - primaries: string; - transferCharacteristics: string; - matrixCoefficients: string; - }; - highReplication?: boolean; - approxDurationMs?: string; - targetDurationSec?: number; - maxDvrDurationSec?: number; - audioSampleRate?: string; - audioChannels?: number; - - // Added by ytdl-core - container: "flv" | "3gp" | "mp4" | "webm" | "ts"; - hasVideo: boolean; - hasAudio: boolean; - codecs: string; - videoCodec?: string; - audioCodec?: string; - - isLive: boolean; - isHLS: boolean; - isDashMPD: boolean; -} - -export type Filter = - | "audioandvideo" - | "videoandaudio" - | "video" - | "videoonly" - | "audio" - | "audioonly" - | ((format: VideoFormat) => boolean); - -export interface GetInfoOptions extends RequestInit { - lang?: string; -} - -export interface ChooseFormatOptions { - quality?: - | "lowest" - | "highest" - | "highestaudio" - | "lowestaudio" - | "highestvideo" - | "lowestvideo" - | string - | number - | string[] - | number[]; - filter?: Filter; - format?: VideoFormat; -} - -export interface DownloadOptions extends GetInfoOptions, ChooseFormatOptions { - range?: { - start?: number; - end?: number; - }; - begin?: string | number | Date; - dlChunkSize?: number; - IPv6Block?: string; - requestOptions?: any; -} - -export interface Thumbnail { - url: string; - width: number; - height: number; -} - -export interface CaptionTrack { - baseUrl: string; - name: { - simpleText: - | "Afrikaans" - | "Albanian" - | "Amharic" - | "Arabic" - | "Armenian" - | "Azerbaijani" - | "Bangla" - | "Basque" - | "Belarusian" - | "Bosnian" - | "Bulgarian" - | "Burmese" - | "Catalan" - | "Cebuano" - | "Chinese (Simplified)" - | "Chinese (Traditional)" - | "Corsican" - | "Croatian" - | "Czech" - | "Danish" - | "Dutch" - | "English" - | "English (auto-generated)" - | "Esperanto" - | "Estonian" - | "Filipino" - | "Finnish" - | "French" - | "Galician" - | "Georgian" - | "German" - | "Greek" - | "Gujarati" - | "Haitian Creole" - | "Hausa" - | "Hawaiian" - | "Hebrew" - | "Hindi" - | "Hmong" - | "Hungarian" - | "Icelandic" - | "Igbo" - | "Indonesian" - | "Irish" - | "Italian" - | "Japanese" - | "Javanese" - | "Kannada" - | "Kazakh" - | "Khmer" - | "Korean" - | "Kurdish" - | "Kyrgyz" - | "Lao" - | "Latin" - | "Latvian" - | "Lithuanian" - | "Luxembourgish" - | "Macedonian" - | "Malagasy" - | "Malay" - | "Malayalam" - | "Maltese" - | "Maori" - | "Marathi" - | "Mongolian" - | "Nepali" - | "Norwegian" - | "Nyanja" - | "Pashto" - | "Persian" - | "Polish" - | "Portuguese" - | "Punjabi" - | "Romanian" - | "Russian" - | "Samoan" - | "Scottish Gaelic" - | "Serbian" - | "Shona" - | "Sindhi" - | "Sinhala" - | "Slovak" - | "Slovenian" - | "Somali" - | "Southern Sotho" - | "Spanish" - | "Spanish (Spain)" - | "Sundanese" - | "Swahili" - | "Swedish" - | "Tajik" - | "Tamil" - | "Telugu" - | "Thai" - | "Turkish" - | "Ukrainian" - | "Urdu" - | "Uzbek" - | "Vietnamese" - | "Welsh" - | "Western Frisian" - | "Xhosa" - | "Yiddish" - | "Yoruba" - | "Zulu" - | string; - }; - vssId: string; - languageCode: - | "af" - | "sq" - | "am" - | "ar" - | "hy" - | "az" - | "bn" - | "eu" - | "be" - | "bs" - | "bg" - | "my" - | "ca" - | "ceb" - | "zh-Hans" - | "zh-Hant" - | "co" - | "hr" - | "cs" - | "da" - | "nl" - | "en" - | "eo" - | "et" - | "fil" - | "fi" - | "fr" - | "gl" - | "ka" - | "de" - | "el" - | "gu" - | "ht" - | "ha" - | "haw" - | "iw" - | "hi" - | "hmn" - | "hu" - | "is" - | "ig" - | "id" - | "ga" - | "it" - | "ja" - | "jv" - | "kn" - | "kk" - | "km" - | "ko" - | "ku" - | "ky" - | "lo" - | "la" - | "lv" - | "lt" - | "lb" - | "mk" - | "mg" - | "ms" - | "ml" - | "mt" - | "mi" - | "mr" - | "mn" - | "ne" - | "no" - | "ny" - | "ps" - | "fa" - | "pl" - | "pt" - | "pa" - | "ro" - | "ru" - | "sm" - | "gd" - | "sr" - | "sn" - | "sd" - | "si" - | "sk" - | "sl" - | "so" - | "st" - | "es" - | "su" - | "sw" - | "sv" - | "tg" - | "ta" - | "te" - | "th" - | "tr" - | "uk" - | "ur" - | "uz" - | "vi" - | "cy" - | "fy" - | "xh" - | "yi" - | "yo" - | "zu" - | string; - kind: string; - rtl?: boolean; - isTranslatable: boolean; -} - -export interface AudioTrack { - captionTrackIndices: number[]; -} - -export interface TranslationLanguage { - languageCode: CaptionTrack["languageCode"]; - languageName: CaptionTrack["name"]; -} - -export interface VideoDetails { - videoId: string; - title: string; - shortDescription: string; - lengthSeconds: string; - keywords?: string[]; - channelId: string; - isOwnerViewing: boolean; - isCrawlable: boolean; - thumbnail: { - thumbnails: Thumbnail[]; - }; - averageRating: number; - allowRatings: boolean; - viewCount: string; - author: string; - isPrivate: boolean; - isUnpluggedCorpus: boolean; - isLiveContent: boolean; -} - -export interface Media { - category: string; - category_url: string; - game?: string; - game_url?: string; - year?: number; - song?: string; - artist?: string; - artist_url?: string; - writers?: string; - licensed_by?: string; - thumbnails: Thumbnail[]; -} - -export interface Author { - id: string; - name: string; - avatar: string; // to remove later - thumbnails?: Thumbnail[]; - verified: boolean; - user?: string; - channel_url: string; - external_channel_url?: string; - user_url?: string; - subscriber_count?: number; -} - -export interface MicroformatRenderer { - thumbnail: { - thumbnails: Thumbnail[]; - }; - embed: { - iframeUrl: string; - flashUrl: string; - width: number; - height: number; - flashSecureUrl: string; - }; - title: { - simpleText: string; - }; - description: { - simpleText: string; - }; - lengthSeconds: string; - ownerProfileUrl: string; - ownerGplusProfileUrl?: string; - externalChannelId: string; - isFamilySafe: boolean; - availableCountries: string[]; - isUnlisted: boolean; - hasYpcMetadata: boolean; - viewCount: string; - category: string; - publishDate: string; - ownerChannelName: string; - liveBroadcastDetails?: { - isLiveNow: boolean; - startTimestamp: string; - }; - uploadDate: string; -} - -export interface Storyboard { - templateUrl: string; - thumbnailWidth: number; - thumbnailHeight: number; - thumbnailCount: number; - interval: number; - columns: number; - rows: number; - storyboardCount: number; -} - -export interface Chapter { - title: string; - start_time: number; -} - -export interface MoreVideoDetails - extends Omit, - Omit { - published: number; - video_url: string; - age_restricted: boolean; - likes: number | null; - dislikes: number | null; - media: Media; - author: Author; - thumbnails: Thumbnail[]; - storyboards: Storyboard[]; - chapters: Chapter[]; - description: string | null; -} - -export interface VideoInfo { - full?: boolean; - iv_load_policy?: string; - iv_allow_in_place_switch?: string; - iv_endscreen_url?: string; - iv_invideo_url?: string; - iv3_module?: string; - rmktEnabled?: string; - uid?: string; - vid?: string; - focEnabled?: string; - baseUrl?: string; - storyboard_spec?: string; - serialized_ad_ux_config?: string; - player_error_log_fraction?: string; - sffb?: string; - ldpj?: string; - videostats_playback_base_url?: string; - innertube_context_client_version?: string; - t?: string; - fade_in_start_milliseconds: string; - timestamp: string; - ad3_module: string; - relative_loudness: string; - allow_below_the_player_companion: string; - eventid: string; - token: string; - atc: string; - cr: string; - apply_fade_on_midrolls: string; - cl: string; - fexp: string[]; - apiary_host: string; - fade_in_duration_milliseconds: string; - fflags: string; - ssl: string; - pltype: string; - enabled_engage_types: string; - hl: string; - is_listed: string; - gut_tag: string; - apiary_host_firstparty: string; - enablecsi: string; - csn: string; - status: string; - afv_ad_tag: string; - idpj: string; - sfw_player_response: string; - account_playback_token: string; - encoded_ad_safety_reason: string; - tag_for_children_directed: string; - no_get_video_log: string; - ppv_remarketing_url: string; - fmt_list: string[][]; - ad_slots: string; - fade_out_duration_milliseconds: string; - instream_long: string; - allow_html5_ads: string; - core_dbp: string; - ad_device: string; - itct: string; - root_ve_type: string; - excluded_ads: string; - aftv: string; - loeid: string; - cver: string; - shortform: string; - dclk: string; - csi_page_type: string; - ismb: string; - gpt_migration: string; - loudness: string; - ad_tag: string; - of: string; - probe_url: string; - vm: string; - afv_ad_tag_restricted_to_instream: string; - gapi_hint_params: string; - cid: string; - c: string; - oid: string; - ptchn: string; - as_launched_in_country: string; - avg_rating: string; - fade_out_start_milliseconds: string; - midroll_prefetch_size: string; - allow_ratings: string; - thumbnail_url: string; - iurlsd: string; - iurlmq: string; - iurlhq: string; - iurlmaxres: string; - ad_preroll: string; - tmi: string; - trueview: string; - host_language: string; - innertube_api_key: string; - show_content_thumbnail: string; - afv_instream_max: string; - innertube_api_version: string; - mpvid: string; - allow_embed: string; - ucid: string; - plid: string; - midroll_freqcap: string; - ad_logging_flag: string; - ptk: string; - vmap: string; - watermark: string[]; - dbp: string; - ad_flags: string; - html5player: string; - formats: VideoFormat[]; - related_videos: RelatedVideo[]; - no_embed_allowed?: boolean; - player_response: { - playabilityStatus: { - status: string; - playableInEmbed: boolean; - miniplayer: { - miniplayerRenderer: { - playbackMode: string; - }; - }; - contextParams: string; - }; - streamingData: { - expiresInSeconds: string; - formats: {}[]; - adaptiveFormats: {}[]; - dashManifestUrl?: string; - hlsManifestUrl?: string; - }; - captions?: { - playerCaptionsRenderer: { - baseUrl: string; - visibility: string; - }; - playerCaptionsTracklistRenderer: { - captionTracks: CaptionTrack[]; - audioTracks: AudioTrack[]; - translationLanguages: TranslationLanguage[]; - defaultAudioTrackIndex: number; - }; - }; - microformat: { - playerMicroformatRenderer: MicroformatRenderer; - }; - videoDetails: VideoDetails; - }; - videoDetails: MoreVideoDetails; -} - -export interface RelatedVideo { - id?: string; - title?: string; - published?: string; - author: Author | "string"; // to remove the `string` part later - ucid?: string; // to remove later - author_thumbnail?: string; // to remove later - short_view_count_text?: string; - view_count?: string; - length_seconds?: number; - video_thumbnail?: string; // to remove later - thumbnails: Thumbnail[]; - richThumbnails: Thumbnail[]; - isLive: boolean; -} +export interface VideoFormat { + itag: number; + url: string; + mimeType?: string; + bitrate?: number; + audioBitrate?: number; + width?: number; + height?: number; + initRange?: { start: string; end: string }; + indexRange?: { start: string; end: string }; + lastModified: string; + contentLength: string; + quality: + | "tiny" + | "small" + | "medium" + | "large" + | "hd720" + | "hd1080" + | "hd1440" + | "hd2160" + | "highres" + | string; + qualityLabel: + | "144p" + | "144p 15fps" + | "144p60 HDR" + | "240p" + | "240p60 HDR" + | "270p" + | "360p" + | "360p60 HDR" + | "480p" + | "480p60 HDR" + | "720p" + | "720p60" + | "720p60 HDR" + | "1080p" + | "1080p60" + | "1080p60 HDR" + | "1440p" + | "1440p60" + | "1440p60 HDR" + | "2160p" + | "2160p60" + | "2160p60 HDR" + | "4320p" + | "4320p60"; + projectionType?: "RECTANGULAR"; + fps?: number; + averageBitrate?: number; + audioQuality?: "AUDIO_QUALITY_LOW" | "AUDIO_QUALITY_MEDIUM"; + colorInfo?: { + primaries: string; + transferCharacteristics: string; + matrixCoefficients: string; + }; + highReplication?: boolean; + approxDurationMs?: string; + targetDurationSec?: number; + maxDvrDurationSec?: number; + audioSampleRate?: string; + audioChannels?: number; + + // Added by ytdl-core + container: "flv" | "3gp" | "mp4" | "webm" | "ts"; + hasVideo: boolean; + hasAudio: boolean; + codecs: string; + videoCodec?: string; + audioCodec?: string; + + isLive: boolean; + isHLS: boolean; + isDashMPD: boolean; +} + +export type Filter = + | "audioandvideo" + | "videoandaudio" + | "video" + | "videoonly" + | "audio" + | "audioonly" + | ((format: VideoFormat) => boolean); + +export interface GetInfoOptions extends RequestInit { + lang?: string; +} + +export interface ChooseFormatOptions { + quality?: + | "lowest" + | "highest" + | "highestaudio" + | "lowestaudio" + | "highestvideo" + | "lowestvideo" + | string + | number + | string[] + | number[]; + filter?: Filter; + format?: VideoFormat; +} + +export interface DownloadOptions extends GetInfoOptions, ChooseFormatOptions { + range?: { + start?: number; + end?: number; + }; + begin?: string | number | Date; + dlChunkSize?: number; + IPv6Block?: string; + requestOptions?: any; +} + +export interface Thumbnail { + url: string; + width: number; + height: number; +} + +export interface CaptionTrack { + baseUrl: string; + name: { + simpleText: + | "Afrikaans" + | "Albanian" + | "Amharic" + | "Arabic" + | "Armenian" + | "Azerbaijani" + | "Bangla" + | "Basque" + | "Belarusian" + | "Bosnian" + | "Bulgarian" + | "Burmese" + | "Catalan" + | "Cebuano" + | "Chinese (Simplified)" + | "Chinese (Traditional)" + | "Corsican" + | "Croatian" + | "Czech" + | "Danish" + | "Dutch" + | "English" + | "English (auto-generated)" + | "Esperanto" + | "Estonian" + | "Filipino" + | "Finnish" + | "French" + | "Galician" + | "Georgian" + | "German" + | "Greek" + | "Gujarati" + | "Haitian Creole" + | "Hausa" + | "Hawaiian" + | "Hebrew" + | "Hindi" + | "Hmong" + | "Hungarian" + | "Icelandic" + | "Igbo" + | "Indonesian" + | "Irish" + | "Italian" + | "Japanese" + | "Javanese" + | "Kannada" + | "Kazakh" + | "Khmer" + | "Korean" + | "Kurdish" + | "Kyrgyz" + | "Lao" + | "Latin" + | "Latvian" + | "Lithuanian" + | "Luxembourgish" + | "Macedonian" + | "Malagasy" + | "Malay" + | "Malayalam" + | "Maltese" + | "Maori" + | "Marathi" + | "Mongolian" + | "Nepali" + | "Norwegian" + | "Nyanja" + | "Pashto" + | "Persian" + | "Polish" + | "Portuguese" + | "Punjabi" + | "Romanian" + | "Russian" + | "Samoan" + | "Scottish Gaelic" + | "Serbian" + | "Shona" + | "Sindhi" + | "Sinhala" + | "Slovak" + | "Slovenian" + | "Somali" + | "Southern Sotho" + | "Spanish" + | "Spanish (Spain)" + | "Sundanese" + | "Swahili" + | "Swedish" + | "Tajik" + | "Tamil" + | "Telugu" + | "Thai" + | "Turkish" + | "Ukrainian" + | "Urdu" + | "Uzbek" + | "Vietnamese" + | "Welsh" + | "Western Frisian" + | "Xhosa" + | "Yiddish" + | "Yoruba" + | "Zulu" + | string; + }; + vssId: string; + languageCode: + | "af" + | "sq" + | "am" + | "ar" + | "hy" + | "az" + | "bn" + | "eu" + | "be" + | "bs" + | "bg" + | "my" + | "ca" + | "ceb" + | "zh-Hans" + | "zh-Hant" + | "co" + | "hr" + | "cs" + | "da" + | "nl" + | "en" + | "eo" + | "et" + | "fil" + | "fi" + | "fr" + | "gl" + | "ka" + | "de" + | "el" + | "gu" + | "ht" + | "ha" + | "haw" + | "iw" + | "hi" + | "hmn" + | "hu" + | "is" + | "ig" + | "id" + | "ga" + | "it" + | "ja" + | "jv" + | "kn" + | "kk" + | "km" + | "ko" + | "ku" + | "ky" + | "lo" + | "la" + | "lv" + | "lt" + | "lb" + | "mk" + | "mg" + | "ms" + | "ml" + | "mt" + | "mi" + | "mr" + | "mn" + | "ne" + | "no" + | "ny" + | "ps" + | "fa" + | "pl" + | "pt" + | "pa" + | "ro" + | "ru" + | "sm" + | "gd" + | "sr" + | "sn" + | "sd" + | "si" + | "sk" + | "sl" + | "so" + | "st" + | "es" + | "su" + | "sw" + | "sv" + | "tg" + | "ta" + | "te" + | "th" + | "tr" + | "uk" + | "ur" + | "uz" + | "vi" + | "cy" + | "fy" + | "xh" + | "yi" + | "yo" + | "zu" + | string; + kind: string; + rtl?: boolean; + isTranslatable: boolean; +} + +export interface AudioTrack { + captionTrackIndices: number[]; +} + +export interface TranslationLanguage { + languageCode: CaptionTrack["languageCode"]; + languageName: CaptionTrack["name"]; +} + +export interface VideoDetails { + videoId: string; + title: string; + shortDescription: string; + lengthSeconds: string; + keywords?: string[]; + channelId: string; + isOwnerViewing: boolean; + isCrawlable: boolean; + thumbnail: { + thumbnails: Thumbnail[]; + }; + averageRating: number; + allowRatings: boolean; + viewCount: string; + author: string; + isPrivate: boolean; + isUnpluggedCorpus: boolean; + isLiveContent: boolean; +} + +export interface Media { + category: string; + category_url: string; + game?: string; + game_url?: string; + year?: number; + song?: string; + artist?: string; + artist_url?: string; + writers?: string; + licensed_by?: string; + thumbnails: Thumbnail[]; +} + +export interface Author { + id: string; + name: string; + avatar: string; // to remove later + thumbnails?: Thumbnail[]; + verified: boolean; + user?: string; + channel_url: string; + external_channel_url?: string; + user_url?: string; + subscriber_count?: number; +} + +export interface MicroformatRenderer { + thumbnail: { + thumbnails: Thumbnail[]; + }; + embed: { + iframeUrl: string; + flashUrl: string; + width: number; + height: number; + flashSecureUrl: string; + }; + title: { + simpleText: string; + }; + description: { + simpleText: string; + }; + lengthSeconds: string; + ownerProfileUrl: string; + ownerGplusProfileUrl?: string; + externalChannelId: string; + isFamilySafe: boolean; + availableCountries: string[]; + isUnlisted: boolean; + hasYpcMetadata: boolean; + viewCount: string; + category: string; + publishDate: string; + ownerChannelName: string; + liveBroadcastDetails?: { + isLiveNow: boolean; + startTimestamp: string; + }; + uploadDate: string; +} + +export interface Storyboard { + templateUrl: string; + thumbnailWidth: number; + thumbnailHeight: number; + thumbnailCount: number; + interval: number; + columns: number; + rows: number; + storyboardCount: number; +} + +export interface Chapter { + title: string; + start_time: number; +} + +export interface MoreVideoDetails + extends Omit, + Omit { + published: number; + video_url: string; + age_restricted: boolean; + likes: number | null; + dislikes: number | null; + media: Media; + author: Author; + thumbnails: Thumbnail[]; + storyboards: Storyboard[]; + chapters: Chapter[]; + description: string | null; +} + +export interface VideoInfo { + full?: boolean; + iv_load_policy?: string; + iv_allow_in_place_switch?: string; + iv_endscreen_url?: string; + iv_invideo_url?: string; + iv3_module?: string; + rmktEnabled?: string; + uid?: string; + vid?: string; + focEnabled?: string; + baseUrl?: string; + storyboard_spec?: string; + serialized_ad_ux_config?: string; + player_error_log_fraction?: string; + sffb?: string; + ldpj?: string; + videostats_playback_base_url?: string; + innertube_context_client_version?: string; + t?: string; + fade_in_start_milliseconds: string; + timestamp: string; + ad3_module: string; + relative_loudness: string; + allow_below_the_player_companion: string; + eventid: string; + token: string; + atc: string; + cr: string; + apply_fade_on_midrolls: string; + cl: string; + fexp: string[]; + apiary_host: string; + fade_in_duration_milliseconds: string; + fflags: string; + ssl: string; + pltype: string; + enabled_engage_types: string; + hl: string; + is_listed: string; + gut_tag: string; + apiary_host_firstparty: string; + enablecsi: string; + csn: string; + status: string; + afv_ad_tag: string; + idpj: string; + sfw_player_response: string; + account_playback_token: string; + encoded_ad_safety_reason: string; + tag_for_children_directed: string; + no_get_video_log: string; + ppv_remarketing_url: string; + fmt_list: string[][]; + ad_slots: string; + fade_out_duration_milliseconds: string; + instream_long: string; + allow_html5_ads: string; + core_dbp: string; + ad_device: string; + itct: string; + root_ve_type: string; + excluded_ads: string; + aftv: string; + loeid: string; + cver: string; + shortform: string; + dclk: string; + csi_page_type: string; + ismb: string; + gpt_migration: string; + loudness: string; + ad_tag: string; + of: string; + probe_url: string; + vm: string; + afv_ad_tag_restricted_to_instream: string; + gapi_hint_params: string; + cid: string; + c: string; + oid: string; + ptchn: string; + as_launched_in_country: string; + avg_rating: string; + fade_out_start_milliseconds: string; + midroll_prefetch_size: string; + allow_ratings: string; + thumbnail_url: string; + iurlsd: string; + iurlmq: string; + iurlhq: string; + iurlmaxres: string; + ad_preroll: string; + tmi: string; + trueview: string; + host_language: string; + innertube_api_key: string; + show_content_thumbnail: string; + afv_instream_max: string; + innertube_api_version: string; + mpvid: string; + allow_embed: string; + ucid: string; + plid: string; + midroll_freqcap: string; + ad_logging_flag: string; + ptk: string; + vmap: string; + watermark: string[]; + dbp: string; + ad_flags: string; + html5player: string; + formats: VideoFormat[]; + related_videos: RelatedVideo[]; + no_embed_allowed?: boolean; + player_response: { + playabilityStatus: { + status: string; + playableInEmbed: boolean; + miniplayer: { + miniplayerRenderer: { + playbackMode: string; + }; + }; + contextParams: string; + }; + streamingData: { + expiresInSeconds: string; + formats: {}[]; + adaptiveFormats: {}[]; + dashManifestUrl?: string; + hlsManifestUrl?: string; + }; + captions?: { + playerCaptionsRenderer: { + baseUrl: string; + visibility: string; + }; + playerCaptionsTracklistRenderer: { + captionTracks: CaptionTrack[]; + audioTracks: AudioTrack[]; + translationLanguages: TranslationLanguage[]; + defaultAudioTrackIndex: number; + }; + }; + microformat: { + playerMicroformatRenderer: MicroformatRenderer; + }; + videoDetails: VideoDetails; + }; + videoDetails: MoreVideoDetails; +} + +export interface RelatedVideo { + id?: string; + title?: string; + published?: string; + author: Author | "string"; // to remove the `string` part later + ucid?: string; // to remove later + author_thumbnail?: string; // to remove later + short_view_count_text?: string; + view_count?: string; + length_seconds?: number; + video_thumbnail?: string; // to remove later + thumbnails: Thumbnail[]; + richThumbnails: Thumbnail[]; + isLive: boolean; +} diff --git a/src/url_utils.ts b/src/url_utils.ts index eb61318..ecb2227 100644 --- a/src/url_utils.ts +++ b/src/url_utils.ts @@ -1,77 +1,77 @@ -/** - * Returns true if given id satifies YouTube's id format. - * - * @param {string} id - * @return {boolean} - */ -const idRegex = /^[a-zA-Z0-9-_]{11}$/; -export const validateID = (id: string) => idRegex.test(id); - -const validQueryDomains = new Set([ - "youtube.com", - "www.youtube.com", - "m.youtube.com", - "music.youtube.com", - "gaming.youtube.com", -]); -const validPathDomains = /^https?:\/\/(youtu\.be\/|(www\.)?youtube.com\/(embed|v|shorts)\/)/; -/** - * Get video ID. - * - * There are a few type of video URL formats. - * - https://www.youtube.com/watch?v=VIDEO_ID - * - https://m.youtube.com/watch?v=VIDEO_ID - * - https://youtu.be/VIDEO_ID - * - https://www.youtube.com/v/VIDEO_ID - * - https://www.youtube.com/embed/VIDEO_ID - * - https://music.youtube.com/watch?v=VIDEO_ID - * - https://gaming.youtube.com/watch?v=VIDEO_ID - */ -export const getURLVideoID = (link: string) => { - const parsed = new URL(link); - let id = parsed.searchParams.get("v"); - if (validPathDomains.test(link) && !id) { - const paths = parsed.pathname.split("/"); - id = paths[paths.length - 1]; - } else if (parsed.hostname && !validQueryDomains.has(parsed.hostname)) { - throw Error("Not a YouTube domain"); - } - if (!id) { - throw Error(`No video id found: ${link}`); - } - id = id.substring(0, 11); - if (!validateID(id)) { - throw TypeError( - `Video id (${id}) does not match expected ` + - `format (${idRegex.toString()})` - ); - } - return id; -}; - -/** - * Gets video ID either from a url or by checking if the given string - * matches the video ID format. - */ -const urlRegex = /^https?:\/\//; -export const getVideoID = (str: string) => { - if (validateID(str)) { - return str; - } else if (urlRegex.test(str)) { - return getURLVideoID(str); - } else { - throw Error(`No video id found: ${str}`); - } -}; - -/** - * Checks wether the input string includes a valid id. - */ -export const validateURL = (string: string) => { - try { - getURLVideoID(string); - return true; - } catch (e) { - return false; - } -}; +/** + * Returns true if given id satifies YouTube's id format. + * + * @param {string} id + * @return {boolean} + */ +const idRegex = /^[a-zA-Z0-9-_]{11}$/; +export const validateID = (id: string) => idRegex.test(id); + +const validQueryDomains = new Set([ + "youtube.com", + "www.youtube.com", + "m.youtube.com", + "music.youtube.com", + "gaming.youtube.com", +]); +const validPathDomains = /^https?:\/\/(youtu\.be\/|(www\.)?youtube.com\/(embed|v|shorts)\/)/; +/** + * Get video ID. + * + * There are a few type of video URL formats. + * - https://www.youtube.com/watch?v=VIDEO_ID + * - https://m.youtube.com/watch?v=VIDEO_ID + * - https://youtu.be/VIDEO_ID + * - https://www.youtube.com/v/VIDEO_ID + * - https://www.youtube.com/embed/VIDEO_ID + * - https://music.youtube.com/watch?v=VIDEO_ID + * - https://gaming.youtube.com/watch?v=VIDEO_ID + */ +export const getURLVideoID = (link: string) => { + const parsed = new URL(link); + let id = parsed.searchParams.get("v"); + if (validPathDomains.test(link) && !id) { + const paths = parsed.pathname.split("/"); + id = paths[paths.length - 1]; + } else if (parsed.hostname && !validQueryDomains.has(parsed.hostname)) { + throw Error("Not a YouTube domain"); + } + if (!id) { + throw Error(`No video id found: ${link}`); + } + id = id.substring(0, 11); + if (!validateID(id)) { + throw TypeError( + `Video id (${id}) does not match expected ` + + `format (${idRegex.toString()})` + ); + } + return id; +}; + +/** + * Gets video ID either from a url or by checking if the given string + * matches the video ID format. + */ +const urlRegex = /^https?:\/\//; +export const getVideoID = (str: string) => { + if (validateID(str)) { + return str; + } else if (urlRegex.test(str)) { + return getURLVideoID(str); + } else { + throw Error(`No video id found: ${str}`); + } +}; + +/** + * Checks wether the input string includes a valid id. + */ +export const validateURL = (string: string) => { + try { + getURLVideoID(string); + return true; + } catch (e) { + return false; + } +}; diff --git a/src/utils.ts b/src/utils.ts index f75e81b..0620378 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,191 +1,191 @@ -/** - * Extract string inbetween another. - */ -export function between( - haystack: string, - left: string | RegExp, - right: string -): string { - let pos; - if (left instanceof RegExp) { - const match = haystack.match(left); - if (!match) { - return ""; - } - pos = match.index! + match[0].length; - } else { - pos = haystack.indexOf(left); - if (pos === -1) { - return ""; - } - pos += left.length; - } - haystack = haystack.slice(pos); - pos = haystack.indexOf(right); - if (pos === -1) { - return ""; - } - haystack = haystack.slice(0, pos); - return haystack; -} - -/** Get a number from an abbreviated number string. */ -export function parseAbbreviatedNumber(str: string) { - const match = str - .replace(",", ".") - .replace(" ", "") - .match(/([\d,.]+)([MK]?)/); - if (match) { - let [, _num, multi] = match; - let num = parseFloat(_num); - return Math.round( - multi === "M" ? num * 1000000 : multi === "K" ? num * 1000 : num - ); - } - return null; -} - -/** Match begin and end braces of input JSON, return only json */ -export function cutAfterJSON(mixedJson: string) { - let open, close; - if (mixedJson[0] === "[") { - open = "["; - close = "]"; - } else if (mixedJson[0] === "{") { - open = "{"; - close = "}"; - } - - if (!open) { - throw new Error( - `Can't cut unsupported JSON (need to begin with [ or { ) but got: ${mixedJson[0]}` - ); - } - - // States if the loop is currently in a string - let isString = false; - - // States if the current character is treated as escaped or not - let isEscaped = false; - - // Current open brackets to be closed - let counter = 0; - - let i; - for (i = 0; i < mixedJson.length; i++) { - // Toggle the isString boolean when leaving/entering string - if (mixedJson[i] === '"' && !isEscaped) { - isString = !isString; - continue; - } - - // Toggle the isEscaped boolean for every backslash - // Reset for every regular character - isEscaped = mixedJson[i] === "\\" && !isEscaped; - - if (isString) continue; - - if (mixedJson[i] === open) { - counter++; - } else if (mixedJson[i] === close) { - counter--; - } - - // All brackets have been closed, thus end of JSON is reached - if (counter === 0) { - // Return the cut JSON - return mixedJson.substr(0, i + 1); - } - } - - // We ran through the whole string and ended up with an unclosed bracket - throw Error("Can't cut unsupported JSON (no matching closing bracket found)"); -} - -/** Checks if there is a playability error. */ -export function playError( - player_response: any, - statuses: string[], - ErrorType = Error -) { - let playability = player_response && player_response.playabilityStatus; - if (playability && statuses.includes(playability.status)) { - return new ErrorType( - playability.reason || (playability.messages && playability.messages[0]) - ); - } - return null; -} - -// eslint-disable-next-line max-len -const IPV6_REGEX = - /^(([0-9a-f]{1,4}:)(:[0-9a-f]{1,4}){1,6}|([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}|([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}|([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}|([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}|([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4})|([0-9a-f]{1,4}:){1,7}(([0-9a-f]{1,4})|:))\/(1[0-1]\d|12[0-8]|\d{1,2})$/; - -/** - * Quick check for a valid IPv6 - * The Regex only accepts a subset of all IPv6 Addresses - * - * @param {string} ip the IPv6 block in CIDR-Notation to test - * @returns {boolean} true if valid - */ -export function isIPv6(ip: string) { - return IPV6_REGEX.test(ip); -} - -/** - * Gets random IPv6 Address from a block - */ -export function getRandomIPv6(ip: string) { - // Start with a fast Regex-Check - if (!isIPv6(ip)) throw Error("Invalid IPv6 format"); - // Start by splitting and normalizing addr and mask - const [rawAddr, rawMask] = ip.split("/"); - let base10Mask = parseInt(rawMask); - if (!base10Mask || base10Mask > 128 || base10Mask < 24) - throw Error("Invalid IPv6 subnet"); - const base10addr = normalizeIP(rawAddr); - // Get random addr to pad with - // using Math.random since we're not requiring high level of randomness - const randomAddr = new Array(8) - .fill(1) - .map(() => Math.floor(Math.random() * 0xffff)); - - // Merge base10addr with randomAddr - const mergedAddr = randomAddr.map((randomItem, idx) => { - // Calculate the amount of static bits - const staticBits = Math.min(base10Mask, 16); - // Adjust the bitmask with the staticBits - base10Mask -= staticBits; - // Calculate the bitmask - // lsb makes the calculation way more complicated - const mask = 0xffff - (2 ** (16 - staticBits) - 1); - // Combine base10addr and random - return (base10addr[idx] & mask) + (randomItem & (mask ^ 0xffff)); - }); - // Return new addr - return mergedAddr.map((x) => x.toString(16)).join(":"); -} -/** - * Normalise an IP Address - * - * @param {string} ip the IPv6 Addr - * @returns {number[]} the 8 parts of the IPv6 as Integers - */ -export function normalizeIP(ip: string) { - // Split by fill position - const parts = ip.split("::").map((x) => x.split(":")); - // Normalize start and end - const partStart = parts[0] || []; - const partEnd = parts[1] || []; - partEnd.reverse(); - // Placeholder for full ip - const fullIP = new Array(8).fill(0); - // Fill in start and end parts - for (let i = 0; i < Math.min(partStart.length, 8); i++) { - fullIP[i] = parseInt(partStart[i], 16) || 0; - } - for (let i = 0; i < Math.min(partEnd.length, 8); i++) { - fullIP[7 - i] = parseInt(partEnd[i], 16) || 0; - } - return fullIP; -} +/** + * Extract string inbetween another. + */ +export function between( + haystack: string, + left: string | RegExp, + right: string +): string { + let pos; + if (left instanceof RegExp) { + const match = haystack.match(left); + if (!match) { + return ""; + } + pos = match.index! + match[0].length; + } else { + pos = haystack.indexOf(left); + if (pos === -1) { + return ""; + } + pos += left.length; + } + haystack = haystack.slice(pos); + pos = haystack.indexOf(right); + if (pos === -1) { + return ""; + } + haystack = haystack.slice(0, pos); + return haystack; +} + +/** Get a number from an abbreviated number string. */ +export function parseAbbreviatedNumber(str: string) { + const match = str + .replace(",", ".") + .replace(" ", "") + .match(/([\d,.]+)([MK]?)/); + if (match) { + let [, _num, multi] = match; + let num = parseFloat(_num); + return Math.round( + multi === "M" ? num * 1000000 : multi === "K" ? num * 1000 : num + ); + } + return null; +} + +/** Match begin and end braces of input JSON, return only json */ +export function cutAfterJSON(mixedJson: string) { + let open, close; + if (mixedJson[0] === "[") { + open = "["; + close = "]"; + } else if (mixedJson[0] === "{") { + open = "{"; + close = "}"; + } + + if (!open) { + throw new Error( + `Can't cut unsupported JSON (need to begin with [ or { ) but got: ${mixedJson[0]}` + ); + } + + // States if the loop is currently in a string + let isString = false; + + // States if the current character is treated as escaped or not + let isEscaped = false; + + // Current open brackets to be closed + let counter = 0; + + let i; + for (i = 0; i < mixedJson.length; i++) { + // Toggle the isString boolean when leaving/entering string + if (mixedJson[i] === '"' && !isEscaped) { + isString = !isString; + continue; + } + + // Toggle the isEscaped boolean for every backslash + // Reset for every regular character + isEscaped = mixedJson[i] === "\\" && !isEscaped; + + if (isString) continue; + + if (mixedJson[i] === open) { + counter++; + } else if (mixedJson[i] === close) { + counter--; + } + + // All brackets have been closed, thus end of JSON is reached + if (counter === 0) { + // Return the cut JSON + return mixedJson.substr(0, i + 1); + } + } + + // We ran through the whole string and ended up with an unclosed bracket + throw Error("Can't cut unsupported JSON (no matching closing bracket found)"); +} + +/** Checks if there is a playability error. */ +export function playError( + player_response: any, + statuses: string[], + ErrorType = Error +) { + let playability = player_response && player_response.playabilityStatus; + if (playability && statuses.includes(playability.status)) { + return new ErrorType( + playability.reason || (playability.messages && playability.messages[0]) + ); + } + return null; +} + +// eslint-disable-next-line max-len +const IPV6_REGEX = + /^(([0-9a-f]{1,4}:)(:[0-9a-f]{1,4}){1,6}|([0-9a-f]{1,4}:){1,2}(:[0-9a-f]{1,4}){1,5}|([0-9a-f]{1,4}:){1,3}(:[0-9a-f]{1,4}){1,4}|([0-9a-f]{1,4}:){1,4}(:[0-9a-f]{1,4}){1,3}|([0-9a-f]{1,4}:){1,5}(:[0-9a-f]{1,4}){1,2}|([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4})|([0-9a-f]{1,4}:){1,7}(([0-9a-f]{1,4})|:))\/(1[0-1]\d|12[0-8]|\d{1,2})$/; + +/** + * Quick check for a valid IPv6 + * The Regex only accepts a subset of all IPv6 Addresses + * + * @param {string} ip the IPv6 block in CIDR-Notation to test + * @returns {boolean} true if valid + */ +export function isIPv6(ip: string) { + return IPV6_REGEX.test(ip); +} + +/** + * Gets random IPv6 Address from a block + */ +export function getRandomIPv6(ip: string) { + // Start with a fast Regex-Check + if (!isIPv6(ip)) throw Error("Invalid IPv6 format"); + // Start by splitting and normalizing addr and mask + const [rawAddr, rawMask] = ip.split("/"); + let base10Mask = parseInt(rawMask); + if (!base10Mask || base10Mask > 128 || base10Mask < 24) + throw Error("Invalid IPv6 subnet"); + const base10addr = normalizeIP(rawAddr); + // Get random addr to pad with + // using Math.random since we're not requiring high level of randomness + const randomAddr = new Array(8) + .fill(1) + .map(() => Math.floor(Math.random() * 0xffff)); + + // Merge base10addr with randomAddr + const mergedAddr = randomAddr.map((randomItem, idx) => { + // Calculate the amount of static bits + const staticBits = Math.min(base10Mask, 16); + // Adjust the bitmask with the staticBits + base10Mask -= staticBits; + // Calculate the bitmask + // lsb makes the calculation way more complicated + const mask = 0xffff - (2 ** (16 - staticBits) - 1); + // Combine base10addr and random + return (base10addr[idx] & mask) + (randomItem & (mask ^ 0xffff)); + }); + // Return new addr + return mergedAddr.map((x) => x.toString(16)).join(":"); +} +/** + * Normalise an IP Address + * + * @param {string} ip the IPv6 Addr + * @returns {number[]} the 8 parts of the IPv6 as Integers + */ +export function normalizeIP(ip: string) { + // Split by fill position + const parts = ip.split("::").map((x) => x.split(":")); + // Normalize start and end + const partStart = parts[0] || []; + const partEnd = parts[1] || []; + partEnd.reverse(); + // Placeholder for full ip + const fullIP = new Array(8).fill(0); + // Fill in start and end parts + for (let i = 0; i < Math.min(partStart.length, 8); i++) { + fullIP[i] = parseInt(partStart[i], 16) || 0; + } + for (let i = 0; i < Math.min(partEnd.length, 8); i++) { + fullIP[7 - i] = parseInt(partEnd[i], 16) || 0; + } + return fullIP; +} diff --git a/src/video.ts b/src/video.ts index 16c86e1..c6dda1c 100644 --- a/src/video.ts +++ b/src/video.ts @@ -1,207 +1,207 @@ -import * as utils from "./utils.ts"; -import * as formatUtils from "./format_util.ts"; -import { parseTimestamp, m3u8stream } from "../deps.ts"; -import { getInfo } from "./info.ts"; -import { VideoStream } from "./stream.ts"; -import { DownloadOptions, VideoFormat } from "./types.ts"; -import { request } from "./request.ts"; - -export interface VideoStreamSource { - stream: VideoStream; - push: CallableFunction; - close: CallableFunction; -} - -function createVideoStreamSource(): VideoStreamSource { - const src: any = { - stream: undefined, - push: () => {}, - close: () => {}, - }; - - src.stream = new VideoStream({ - start: (controller) => { - src.controller = controller; +import * as utils from "./utils.ts"; +import * as formatUtils from "./format_util.ts"; +import { parseTimestamp, m3u8stream } from "../deps.ts"; +import { getInfo } from "./info.ts"; +import { VideoStream } from "./stream.ts"; +import { DownloadOptions, VideoFormat } from "./types.ts"; +import { request } from "./request.ts"; + +export interface VideoStreamSource { + stream: VideoStream; + push: CallableFunction; + close: CallableFunction; +} + +function createVideoStreamSource(): VideoStreamSource { + const src: any = { + stream: undefined, + push: () => {}, + close: () => {}, + }; + + src.stream = new VideoStream({ + start: (controller) => { + src.controller = controller; src.push = (data: Uint8Array) => { - if (src.closed) return; + if (src.closed) return; try { controller.enqueue(data); } catch(_e) { src.closed = true; - } - }; + } + }; src.close = () => { - if (src.closed) return; - try { - controller.close(); + if (src.closed) return; + try { + controller.close(); } catch (_e) { // ignore } finally { src.closed = true; - } - }; - }, - }); - - return src; -} - -async function downloadFromInfoInto( - { stream, push, close }: VideoStreamSource, - info: any, - options: DownloadOptions = {}, -) { - let err = utils.playError(info.player_response, [ - "UNPLAYABLE", - "LIVE_STREAM_OFFLINE", - "LOGIN_REQUIRED", - ]); - if (err) { - stream.cancel(err); - return; - } - - if (!info.formats.length) { - stream.cancel(new Error("This video is unavailable")); - return; - } - - let format: VideoFormat; - try { - format = formatUtils.chooseFormat(info.formats, options); - } catch (e) { - stream.cancel(e); - return; - } - - stream.info = info; - stream.format = format; - - if (stream.locked) return; - - let contentLength: number, - downloaded = 0; - - const ondata = async (chunk: Uint8Array) => { - downloaded += chunk.length; - await push(chunk); - }; - - if (options.IPv6Block) { - options.requestOptions = Object.assign({}, options.requestOptions, { - family: 6, - localAddress: utils.getRandomIPv6(options.IPv6Block), - }); - } - - const dlChunkSize = options.dlChunkSize || 1024 * 1024 * 10; - let req: Response; - let shouldEnd = true; - if (format.isHLS || format.isDashMPD) { - const begin = options.begin || (format.isLive && Date.now()); - const req = m3u8stream(format.url, { - chunkReadahead: +info.live_chunk_readahead, - begin: begin.toString(), - requestOptions: options.requestOptions, - parser: format.isDashMPD ? "dash-mpd" : "m3u8", - id: format.itag.toString(), - }); - req.addListener("data", async (chunk) => { - await push(chunk); - }); - req.addListener("end", async () => { - await close(); - }); - } else { - const requestOptions = Object.assign({}, options, { - maxReconnects: 6, - maxRetries: 3, - backoff: { inc: 500, max: 10000 }, - }); - - let shouldBeChunked = dlChunkSize !== 0 && - (!format.hasAudio || !format.hasVideo); - - if (shouldBeChunked) { - let start = (options.range && options.range.start) || 0; - let end = start + dlChunkSize; - const rangeEnd = options.range && options.range.end; - - contentLength = options.range - ? (rangeEnd ? rangeEnd + 1 : parseInt(format.contentLength)) - start - : parseInt(format.contentLength); - - stream.total = contentLength; - - const getNextChunk = async () => { - if (!rangeEnd && end >= contentLength) end = 0; - if (rangeEnd && end > rangeEnd) end = rangeEnd; - shouldEnd = !end || end === rangeEnd; - - requestOptions.headers = Object.assign({}, requestOptions.headers, { - Range: `bytes=${start}-${end || ""}`, - }); - - req = await request(format.url, requestOptions); - - for await (const chunk of req.body!) { - stream.downloaded += chunk.length; - await ondata(chunk); - } - - if (end && end !== rangeEnd) { - start = end + 1; - end += dlChunkSize; - await getNextChunk(); - } - - await close(); - }; - - getNextChunk(); - } else { - // Audio only and video only formats don't support begin - if (options.begin) { - format.url += `&begin=${ - parseTimestamp( - typeof options.begin === "object" - ? options.begin.getTime() - : options.begin, - ) - }`; - } - if (options.range && (options.range.start || options.range.end)) { - requestOptions.headers = Object.assign({}, requestOptions.headers, { - Range: `bytes=${options.range.start || "0"}-${options.range.end || - ""}`, - }); - } - req = await request(format.url, requestOptions); - contentLength = parseInt(format.contentLength); - - stream.total = contentLength; - - (async () => { - for await (const chunk of req.body!) { - stream.downloaded += chunk.length; - await ondata(chunk); - } - await close(); - })(); - } - } -} - -export async function downloadFromInfo( - info: any, - options: DownloadOptions = {}, -) { - const src = createVideoStreamSource(); - await downloadFromInfoInto(src, info, options); - return src.stream; -} - -export async function ytdl(id: string, options: DownloadOptions = {}) { - const info = await getInfo(id, options); - return await downloadFromInfo(info, options); -} + } + }; + }, + }); + + return src; +} + +async function downloadFromInfoInto( + { stream, push, close }: VideoStreamSource, + info: any, + options: DownloadOptions = {}, +) { + let err = utils.playError(info.player_response, [ + "UNPLAYABLE", + "LIVE_STREAM_OFFLINE", + "LOGIN_REQUIRED", + ]); + if (err) { + stream.cancel(err); + return; + } + + if (!info.formats.length) { + stream.cancel(new Error("This video is unavailable")); + return; + } + + let format: VideoFormat; + try { + format = formatUtils.chooseFormat(info.formats, options); + } catch (e) { + stream.cancel(e); + return; + } + + stream.info = info; + stream.format = format; + + if (stream.locked) return; + + let contentLength: number, + downloaded = 0; + + const ondata = async (chunk: Uint8Array) => { + downloaded += chunk.length; + await push(chunk); + }; + + if (options.IPv6Block) { + options.requestOptions = Object.assign({}, options.requestOptions, { + family: 6, + localAddress: utils.getRandomIPv6(options.IPv6Block), + }); + } + + const dlChunkSize = options.dlChunkSize || 1024 * 1024 * 10; + let req: Response; + let shouldEnd = true; + if (format.isHLS || format.isDashMPD) { + const begin = options.begin || (format.isLive && Date.now()); + const req = m3u8stream(format.url, { + chunkReadahead: +info.live_chunk_readahead, + begin: begin.toString(), + requestOptions: options.requestOptions, + parser: format.isDashMPD ? "dash-mpd" : "m3u8", + id: format.itag.toString(), + }); + req.addListener("data", async (chunk) => { + await push(chunk); + }); + req.addListener("end", async () => { + await close(); + }); + } else { + const requestOptions = Object.assign({}, options, { + maxReconnects: 6, + maxRetries: 3, + backoff: { inc: 500, max: 10000 }, + }); + + let shouldBeChunked = dlChunkSize !== 0 && + (!format.hasAudio || !format.hasVideo); + + if (shouldBeChunked) { + let start = (options.range && options.range.start) || 0; + let end = start + dlChunkSize; + const rangeEnd = options.range && options.range.end; + + contentLength = options.range + ? (rangeEnd ? rangeEnd + 1 : parseInt(format.contentLength)) - start + : parseInt(format.contentLength); + + stream.total = contentLength; + + const getNextChunk = async () => { + if (!rangeEnd && end >= contentLength) end = 0; + if (rangeEnd && end > rangeEnd) end = rangeEnd; + shouldEnd = !end || end === rangeEnd; + + requestOptions.headers = Object.assign({}, requestOptions.headers, { + Range: `bytes=${start}-${end || ""}`, + }); + + req = await request(format.url, requestOptions); + + for await (const chunk of req.body!) { + stream.downloaded += chunk.length; + await ondata(chunk); + } + + if (end && end !== rangeEnd) { + start = end + 1; + end += dlChunkSize; + await getNextChunk(); + } + + await close(); + }; + + getNextChunk(); + } else { + // Audio only and video only formats don't support begin + if (options.begin) { + format.url += `&begin=${ + parseTimestamp( + typeof options.begin === "object" + ? options.begin.getTime() + : options.begin, + ) + }`; + } + if (options.range && (options.range.start || options.range.end)) { + requestOptions.headers = Object.assign({}, requestOptions.headers, { + Range: `bytes=${options.range.start || "0"}-${options.range.end || + ""}`, + }); + } + req = await request(format.url, requestOptions); + contentLength = parseInt(format.contentLength); + + stream.total = contentLength; + + (async () => { + for await (const chunk of req.body!) { + stream.downloaded += chunk.length; + await ondata(chunk); + } + await close(); + })(); + } + } +} + +export async function downloadFromInfo( + info: any, + options: DownloadOptions = {}, +) { + const src = createVideoStreamSource(); + await downloadFromInfoInto(src, info, options); + return src.stream; +} + +export async function ytdl(id: string, options: DownloadOptions = {}) { + const info = await getInfo(id, options); + return await downloadFromInfo(info, options); +} diff --git a/test.ts b/test.ts index 614bb04..8ad9391 100644 --- a/test.ts +++ b/test.ts @@ -1,14 +1,14 @@ -import ytdl from "./mod.ts"; - -const stream = await ytdl("vRXZj0DzXIA"); -console.log("Size:", stream.total); - -const chunks: Uint8Array[] = []; - -for await (const chunk of stream) { - chunks.push(chunk); -} - -const blob = new Blob(chunks); -console.log("Saving as video.mp4..."); -await Deno.writeFile("video.mp4", new Uint8Array(await blob.arrayBuffer())); +import ytdl from "./mod.ts"; + +const stream = await ytdl("9EtbggRB51g"); +console.log("Size:", stream.total); + +const chunks: Uint8Array[] = []; + +for await (const chunk of stream) { + chunks.push(chunk); +} + +const blob = new Blob(chunks); +console.log("Saving as video.mp4..."); +await Deno.writeFile("video.mp4", new Uint8Array(await blob.arrayBuffer()));