a
    h	G                     @   s   d dl Z d dlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ G dd deZG dd	 d	eZG d
d deZG dd deZG dd deZdS )    N   )InfoExtractor   )ExtractorError
clean_htmldetermine_extextract_attributesfloat_or_noneget_elements_html_by_classint_or_nonemerge_dictsmimetype2extparse_iso8601
remove_endremove_startstr_or_nonetraverse_objurl_or_nonec                   @   sB   e Zd ZedZdZdZdZdd Z	dd Z
d	d
 Zdd ZdS )NYTimesBaseIEz$36dd619a-56dc-595b-9e09-37f4152c7b5da  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQABz/https://samizdat-graphql.nytimes.com/graphql/v2a  query VideoQuery($id: String!) {
  video(id: $id) {
    ... on Video {
      bylines {
        renderedRepresentation
      }
      duration
      firstPublished
      promotionalHeadline
      promotionalMedia {
        ... on Image {
          crops {
            name
            renditions {
              name
              width
              height
              url
            }
          }
        }
      }
      renditions {
        type
        width
        height
        url
        bitrate
      }
      summary
    }
  }
}c                 C   sr   t | jd}t ||}t| j| j|dtj| jdd| iddd	 dd	d
| j
ddddddthfppi S )Nvideoz!Downloading JSON from GraphQL APIidznyt://video/)query	variables),:)
separatorszapplication/jsonZvhszv3.52.21zhttps://nytimes.com)zContent-TypezNyt-App-TypezNyt-App-Versionz	Nyt-TokenZOriginF)dataheadersfatalr   )uuiduuid5_DNS_NAMESPACEr   Z_download_json_GRAPHQL_APIjsondumps_GRAPHQL_QUERYencode_TOKENdict)selfmedia_idZ
video_uuidZ
media_uuid r+   M/mnt/pikpak/tmp/myenv/lib/python3.9/site-packages/yt_dlp/extractor/nytimes.py	_call_api>   s*    


zNYTimesBaseIE._call_apic                 C   s*   t |dd ddthfdthfdfd dS )Nc                 S   s   t |d S )Nurl)r   )_vr+   r+   r,   <lambda>P       z3NYTimesBaseIE._extract_thumbnails.<locals>.<lambda>r.   widthheight)r.   r3   r4   default)r   r   )r)   thumbsr+   r+   r,   _extract_thumbnailsO   s    
z!NYTimesBaseIE._extract_thumbnailsc                 C   s&  g }g }i }t |ddthfD ]}|d}|d}|r|dks||v rLq|| t|dpjt|}	|	dkr| j||dd	|pd
dd\}
}||
 | j||d q|	dkrqq||||dp|dt	|dt	|dt |ddt	hfddt	|ddpd |	d q||fS )N
renditions.r.   typer7   mimetypem3u8mp4Zm3u8_nativehlsF)m3u8_idr   )targetZmpdZvideoencodingZvideo_codecr3   r4   )	file_sizeZfileSize)Nvalueget_allZbitrate  )r.   	format_idvcodecr3   r4   filesizeZtbrext)
r   r(   getappendr   r   #_extract_m3u8_formats_and_subtitlesextendZ_merge_subtitlesr   )r)   video_idZcontent_media_jsonurlsformats	subtitlesr   Z	video_urlrF   rI   Z	m3u8_fmtsZ	m3u8_subsr+   r+   r,   _extract_formats_and_subtitlesV   sD    





z,NYTimesBaseIE._extract_formats_and_subtitlesc                 C   s|   |  |}| ||\}}||d|dt|dt|ddddt|dd	d
dd hf||| t|dd	S )NZpromotionalHeadlinesummaryfirstPublisheddurationrE   Zscalez, bylines.renderedRepresentationc                 S   s
   t | dS )NzBy )r   xr+   r+   r,   r1      r2   z.NYTimesBaseIE._extract_video.<locals>.<lambda>promotionalMediacrops.r9   .)	r   titledescription	timestamprU   creatorrP   rQ   
thumbnails)r-   rR   rJ   r   r	   joinr   r8   )r)   r*   r   rP   rQ   r+   r+   r,   _extract_videox   s     
zNYTimesBaseIE._extract_videoN)__name__
__module____qualname__r   UUIDr!   r'   r"   r%   r-   r8   rR   rd   r+   r+   r+   r,   r      s   
""r   c                   @   sH   e Zd ZdZdgZddddddd	d
dddd	ddddgZdd ZdS )	NYTimesIEzhttps?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)zl<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>zshttp://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263Z a553aa344014e3723d33893d89d4defcZ100000002847155r=   z Verbatim: What Is a Photocopier?z$md5:93603dada88ddbda9395632fdc5da260it]SZ20140428zBrett Weinerz're:https?://\w+\.nyt.com/images/.+\.jpgi  )	r   rI   r^   r_   r`   upload_datera   	thumbnailrU   r.   md5	info_dictzJhttp://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.htmlTr.   Zonly_matchingc                 C   s   |  |}| |S )N)	_match_idrd   )r)   r.   rN   r+   r+   r,   _real_extract   s    
zNYTimesIE._real_extractN)re   rf   rg   
_VALID_URLZ_EMBED_REGEX_TESTSrq   r+   r+   r+   r,   ri      s(   ri   c                   @   s   e Zd ZdZdddddddd	d
dddd
ddddddddddddgddd	dddddd d!d"d#d$d%gdd&d'
d(d)id*d+d,d-d.d/d0d1gd2d3d4d5d6d7d8d9d:d;gd<d=d>d?d5d@dAdBgZdCdD ZdEdF ZdGS )HNYTimesArticleIEzphttps?://(?:www\.)?nytimes\.com/\d{4}/\d{2}/\d{2}/(?!books|podcasts)[^/?#]+/(?:\w+/)?(?P<id>[^./?#]+)(?:\.html)?zhttp://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0Z 3eb5ddb1d6f86254fe4f233826778737Z100000003628438r=   u0   One Company’s New Minimum Wage: $70,000 a Yearz$md5:89ba9ab67ca767bb92bf823d1f138433i-UZ20150414zMatthew WilliamszPatricia Cohen're:https?://\w+\.nyt.com/images/.*\.jpgg     ]@)
r   rI   r^   r_   r`   rj   uploaderra   rk   rU   z%HTTP Error 500: Internal Server Error)r.   rm   rn   skipzMhttps://www.nytimes.com/2023/09/29/health/mosquitoes-genetic-engineering.htmlZ 2365b3555c8aa7f4dd34ca735ad02e6aZ100000009110381Zmp3z<The Gamble: Can Genetically Modified Mosquitoes End Disease?z$md5:9ff8b47acbaf7f3ca8c732f5c815be2eieZ20230929zStephanie NolenzNatalija Gormalovai*  )	r   rI   r^   r_   r`   rj   creatorsrk   rU   rl   zThttps://www.nytimes.com/2023/11/29/business/dealbook/kamala-harris-biden-voters.htmlZ100000009202270uU   Kamala Harris Defends Biden Policies, but Says ‘More Work’ Needed to Reach Votersz$md5:de4212a7e19bb89e4fb14210ca915f1figeZ20231129zBy The New York TimeszKatie RogersgMbhX@)
r   rI   r^   r_   r`   rj   rv   rx   rk   rU   Zskip_downloadr<   )r.   rm   rn   paramszOhttps://www.nytimes.com/2023/12/02/business/air-traffic-controllers-safety.htmlzair-traffic-controllers-safetyzHDrunk and Asleep on the Job: Air Traffic Controllers Pushed to the Brinkz$md5:549e5a5e935bf7d048be53ba3d2c863dZ20231202zEmily SteelzSydney Emberi` ke)r   r^   r_   rj   rx   r`      r.   rn   playlist_countzFhttps://www.nytimes.com/2025/04/30/well/move/hip-mobility-routine.htmlzhip-mobility-routinez!Tight Hips? These Moves Can Help.zRSitting all day is hard on your hips. Try this simple routine for better mobility.zAlyssa AgeszTheodore TaeihZ20250430)r   r^   r_   rx   r`   rj      zShttps://www.nytimes.com/2023/12/02/business/media/netflix-squid-game-challenge.htmlTro   c              	   C   s   t |dthfdddthfd dtddhfdthfffd	thfd
thfddd}| |d|\}}t |dthf}|s|r||dd i || 	t |d||dS )NsourceIdrW   .rX   rU   rE   rV   lengthrT   ZpodcastSeries)r   rv   rU   r`   ZseriesFrC   r   ZfileUrlnone)r.   rG   r[   )rb   rP   rQ   )
r   strr	   r   r   rR   rJ   r   rK   r8   )r)   blockdetailsrP   rQ   r.   r+   r+   r,   _extract_content_from_block   s,    z,NYTimesArticleIE._extract_content_from_blockc                 C   st  |  |}| j||dd}| jd|d|dd dd d	 d
 }|d d }g }dd }t|dd|tf }r|d|d || |t|d|f |st	dt
| |dt|dddddddthfddp| ddg|t|dthft|dddddthf| t|dd}	g }
|D ]}|
t| ||	 q$t|
dkrd| j|
|fi |	S d |i|
d! S )"NT)Zimpersonatezwindow\.__preloadedData\s*=zmedia detailsc                 S   s   |  ddS )NZ	undefinednull)replacerY   r+   r+   r,   r1     r2   z0NYTimesArticleIE._real_extract.<locals>.<lambda>)Ztransform_sourceZinitialDatar   articleZsprinkledBodycontentc                 S   s   | dko|d dv S )NZmediaZ
__typename)ZVideoZAudior+   )kr0   r+   r+   r,   r1     r2   .Z	ledeMediar~   z/Unable to extract any media blocks from webpagez - The New York TimesrS   textFrC   og:descriptiontwitter:descriptionrT   rW   rx   ZdisplayName)r\   Z
assetCrops.r9   .)r^   r_   r`   rx   rb   r   r   r   )rp   _download_webpageZ_search_jsonr   any
setdefaultrJ   rK   rM   r   r   Z_html_extract_titler   _html_search_metar   r8   r   r   lenplaylist_result)r)   r.   page_idwebpageZart_jsonr   blocksZblock_filterZlede_media_blockZcommon_infoentriesr   r+   r+   r,   rq     sV    

zNYTimesArticleIE._real_extractN)re   rf   rg   rr   rs   r   rq   r+   r+   r+   r,   rt      s   Ort   c                   @   sd   e Zd ZdZdZddddddd	d
ddddddddddd	dddddddddgZdd ZdS ) NYTimesCookingIEZNYTimesCookingGuidez4https?://cooking\.nytimes\.com/guides/(?P<id>[\w-]+)z:https://cooking.nytimes.com/guides/13-how-to-cook-a-turkeyz13-how-to-cook-a-turkeyzHow to Cook a Turkeyz$md5:726cfd3f9b161bdf5c279879e8050ca0)r   r^   r_   r   r{   z@https://cooking.nytimes.com/guides/50-how-to-make-mac-and-cheeseZ 64415805fe0b8640fce6b0b9def5989aZ100000005835845r=   zHow to Make Mac and Cheesez$md5:b8f2f33ec1fb7523b21367147c9594f1i`ZZ20180405gQ#@zAlison Romanru   )	r   rI   r^   r_   r`   rj   rU   ra   rk   rl   z9https://cooking.nytimes.com/guides/20-how-to-frost-a-cakez20-how-to-frost-a-cakezHow to Frost a Cakez$md5:a31fe3b98a8ce7b98aae097730c269cd   )r.   rm   rn   r|   c              	   C   s   |  |}| ||}| ddg|}| ddg|}| d|d}ttd|dthd	f}|r|| | t	| j
||||S i | 
|||| jd
|dd ddS )Nzog:titleztwitter:titler   r   z#data-video-player-id="(\d+)"></div>z
lead videoz
video-item.zdata-video-idz*<span itemprop="author">([^<]+)</span></p>authorr5   )r^   r_   ra   )rp   r   r   Z_search_regexr   r
   r   rK   r   maprd   )r)   r.   r   r   r^   r_   Zlead_video_idZ	media_idsr+   r+   r,   rq   ]  s0    

zNYTimesCookingIE._real_extractN)re   rf   rg   ZIE_NAMErr   rs   rq   r+   r+   r+   r,   r   8  s@   
"r   c                   @   sp   e Zd ZdZdddddddd	d
dddddddddddddddddddddddddddgZdd  Zd!S )"NYTimesCookingRecipeIEz2https?://cooking\.nytimes\.com/recipes/(?P<id>\d+)z?https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tartZ 579e83bbe8e61e9de67f80edba8a78a8Z1017817r=   zCranberry Curd Tartz$md5:ad77a3fc321db636256d4343c5742152iKVZ20151118zDavid Tanisru   )r   rI   r^   r_   r`   rj   ra   rk   rl   zKhttps://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookiesZ 58df35998241dcf0620e99e646331b42Z1024781zNeapolitan Checkerboard Cookiesz$md5:ba12394c585ababea951cb6d2fcc6631igeZ20231130zSue Liz:https://cooking.nytimes.com/recipes/1019516-overnight-oatsZ 2fe7965a3adc899913b8e25ada360823Z1019516i +\z$md5:8856ce10239161bd2596ac335b9f9bfbZ20190102zOvernight OatszGenevieve Ko)r   rI   r`   r_   rj   r^   ra   rk   c                 C   s   |  |}| ||}| ||d d d }| j|d |ddd\}}i t|dthfd	thfd
thfdthfddthfd||dd t|ddddt	hfD dS )NpropsZ	pagePropsZrecipeZvideoSrcr=   r>   )r?   r   r^   ZtopnoteZpublishedAtZcontentAttributionZ
cardByline)r   r^   r_   r`   ra   c                 S   s   g | ]}d |iqS )r.   r+   ).0Z	thumb_urlr+   r+   r,   
<listcomp>  r2   z8NYTimesCookingRecipeIE._real_extract.<locals>.<listcomp>imager]   .)rP   rQ   rb   )
rp   r   Z_search_nextjs_datarL   r   r   r   r   r   r   )r)   r.   r   r   Zrecipe_datarP   rQ   r+   r+   r,   rq     s*    


z$NYTimesCookingRecipeIE._real_extractN)re   rf   rg   rr   rs   rq   r+   r+   r+   r,   r   v  sN   )r   )r#   r   commonr   utilsr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   ri   rt   r   r   r+   r+   r+   r,   <module>   s   Ds >