youtube-dl.ts 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267
  1. import { truncate } from 'lodash'
  2. import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers/constants'
  3. import { logger } from './logger'
  4. import { generateVideoImportTmpPath } from './utils'
  5. import { join } from 'path'
  6. import { root } from './core-utils'
  7. import { ensureDir, writeFile, remove } from 'fs-extra'
  8. import * as request from 'request'
  9. import { createWriteStream } from 'fs'
  10. export type YoutubeDLInfo = {
  11. name?: string
  12. description?: string
  13. category?: number
  14. licence?: number
  15. nsfw?: boolean
  16. tags?: string[]
  17. thumbnailUrl?: string
  18. originallyPublishedAt?: Date
  19. }
  20. const processOptions = {
  21. maxBuffer: 1024 * 1024 * 10 // 10MB
  22. }
  23. function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
  24. return new Promise<YoutubeDLInfo>(async (res, rej) => {
  25. const args = opts || [ '-j', '--flat-playlist' ]
  26. const youtubeDL = await safeGetYoutubeDL()
  27. youtubeDL.getInfo(url, args, processOptions, (err, info) => {
  28. if (err) return rej(err)
  29. if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
  30. const obj = buildVideoInfo(normalizeObject(info))
  31. if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
  32. return res(obj)
  33. })
  34. })
  35. }
  36. function downloadYoutubeDLVideo (url: string, timeout: number) {
  37. const path = generateVideoImportTmpPath(url)
  38. let timer
  39. logger.info('Importing youtubeDL video %s', url)
  40. const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
  41. if (process.env.FFMPEG_PATH) {
  42. options.push('--ffmpeg-location')
  43. options.push(process.env.FFMPEG_PATH)
  44. }
  45. return new Promise<string>(async (res, rej) => {
  46. const youtubeDL = await safeGetYoutubeDL()
  47. youtubeDL.exec(url, options, processOptions, err => {
  48. clearTimeout(timer)
  49. if (err) {
  50. remove(path)
  51. .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
  52. return rej(err)
  53. }
  54. return res(path)
  55. })
  56. timer = setTimeout(async () => {
  57. await remove(path)
  58. return rej(new Error('YoutubeDL download timeout.'))
  59. }, timeout)
  60. })
  61. }
  62. // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
  63. // We rewrote it to avoid sync calls
  64. async function updateYoutubeDLBinary () {
  65. logger.info('Updating youtubeDL binary.')
  66. const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
  67. const bin = join(binDirectory, 'youtube-dl')
  68. const detailsPath = join(binDirectory, 'details')
  69. const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
  70. await ensureDir(binDirectory)
  71. return new Promise(res => {
  72. request.get(url, { followRedirect: false }, (err, result) => {
  73. if (err) {
  74. logger.error('Cannot update youtube-dl.', { err })
  75. return res()
  76. }
  77. if (result.statusCode !== 302) {
  78. logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
  79. return res()
  80. }
  81. const url = result.headers.location
  82. const downloadFile = request.get(url)
  83. const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
  84. downloadFile.on('response', result => {
  85. if (result.statusCode !== 200) {
  86. logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
  87. return res()
  88. }
  89. downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
  90. })
  91. downloadFile.on('error', err => {
  92. logger.error('youtube-dl update error.', { err })
  93. return res()
  94. })
  95. downloadFile.on('end', () => {
  96. const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
  97. writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
  98. if (err) {
  99. logger.error('youtube-dl update error: cannot write details.', { err })
  100. return res()
  101. }
  102. logger.info('youtube-dl updated to version %s.', newVersion)
  103. return res()
  104. })
  105. })
  106. })
  107. })
  108. }
  109. async function safeGetYoutubeDL () {
  110. let youtubeDL
  111. try {
  112. youtubeDL = require('youtube-dl')
  113. } catch (e) {
  114. // Download binary
  115. await updateYoutubeDLBinary()
  116. youtubeDL = require('youtube-dl')
  117. }
  118. return youtubeDL
  119. }
  120. function buildOriginallyPublishedAt (obj: any) {
  121. let originallyPublishedAt: Date = null
  122. const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
  123. if (uploadDateMatcher) {
  124. originallyPublishedAt = new Date()
  125. originallyPublishedAt.setHours(0, 0, 0, 0)
  126. const year = parseInt(uploadDateMatcher[1], 10)
  127. // Month starts from 0
  128. const month = parseInt(uploadDateMatcher[2], 10) - 1
  129. const day = parseInt(uploadDateMatcher[3], 10)
  130. originallyPublishedAt.setFullYear(year, month, day)
  131. }
  132. return originallyPublishedAt
  133. }
  134. // ---------------------------------------------------------------------------
  135. export {
  136. updateYoutubeDLBinary,
  137. downloadYoutubeDLVideo,
  138. getYoutubeDLInfo,
  139. safeGetYoutubeDL,
  140. buildOriginallyPublishedAt
  141. }
  142. // ---------------------------------------------------------------------------
  143. function normalizeObject (obj: any) {
  144. const newObj: any = {}
  145. for (const key of Object.keys(obj)) {
  146. // Deprecated key
  147. if (key === 'resolution') continue
  148. const value = obj[key]
  149. if (typeof value === 'string') {
  150. newObj[key] = value.normalize()
  151. } else {
  152. newObj[key] = value
  153. }
  154. }
  155. return newObj
  156. }
  157. function buildVideoInfo (obj: any) {
  158. return {
  159. name: titleTruncation(obj.title),
  160. description: descriptionTruncation(obj.description),
  161. category: getCategory(obj.categories),
  162. licence: getLicence(obj.license),
  163. nsfw: isNSFW(obj),
  164. tags: getTags(obj.tags),
  165. thumbnailUrl: obj.thumbnail || undefined,
  166. originallyPublishedAt: buildOriginallyPublishedAt(obj)
  167. }
  168. }
  169. function titleTruncation (title: string) {
  170. return truncate(title, {
  171. 'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
  172. 'separator': /,? +/,
  173. 'omission': ' […]'
  174. })
  175. }
  176. function descriptionTruncation (description: string) {
  177. if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
  178. return truncate(description, {
  179. 'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
  180. 'separator': /,? +/,
  181. 'omission': ' […]'
  182. })
  183. }
  184. function isNSFW (info: any) {
  185. return info.age_limit && info.age_limit >= 16
  186. }
  187. function getTags (tags: any) {
  188. if (Array.isArray(tags) === false) return []
  189. return tags
  190. .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
  191. .map(t => t.normalize())
  192. .slice(0, 5)
  193. }
  194. function getLicence (licence: string) {
  195. if (!licence) return undefined
  196. if (licence.indexOf('Creative Commons Attribution') !== -1) return 1
  197. return undefined
  198. }
  199. function getCategory (categories: string[]) {
  200. if (!categories) return undefined
  201. const categoryString = categories[0]
  202. if (!categoryString || typeof categoryString !== 'string') return undefined
  203. if (categoryString === 'News & Politics') return 11
  204. for (const key of Object.keys(VIDEO_CATEGORIES)) {
  205. const category = VIDEO_CATEGORIES[key]
  206. if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
  207. }
  208. return undefined
  209. }