a
    hT                     @   s   d dl Zd dlZd dlZd dlZd dlZd dlZd dlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZ ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZ G dd deZG dd de	ZdS )    N   )FileDownloader)HttpFD   )aes_cbc_decrypt_bytesunpad_pkcs7)Request)	HTTPErrorIncompleteRead)DownloadErrorRetryManagertraverse_obj)HTTPHeaderDict)ProgressCalculatorc                   @   s   e Zd Zdd ZeZdS )HttpQuietDownloaderc                 O   s   d S N )selfargsZkargsr   r   O/mnt/pikpak/tmp/myenv/lib/python3.9/site-packages/yt_dlp/downloader/fragment.py	to_screen   s    zHttpQuietDownloader.to_screenN)__name__
__module____qualname__r   Zto_console_titler   r   r   r   r      s   r   c                   @   s   e Zd ZdZdd Zd*ddZdd Zd	d
 Zdd Zdd Z	dd Z
d+ddZdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ d%d$ ddd&d'd(d)ZdS ),
FragmentFDa  
    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).

    Available options:

    fragment_retries:   Number of times to retry a fragment for HTTP error
                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
    skip_unavailable_fragments:
                        Skip unavailable fragments (DASH and hlsnative only)
    keep_fragments:     Keep downloaded fragments on disk after downloading is
                        finished
    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
    _no_ytdl_file:      Don't use .ytdl file

    For each incomplete fragment download yt-dlp keeps on disk a special
    bookkeeping file with download state and metadata (in future such files will
    be used for any incomplete download handled by yt-dlp). This file is
    used to properly handle resuming, check download file consistency and detect
    potential errors. The file has a .ytdl extension and represents a standard
    JSON file of the following format:

    extractor:
        Dictionary of extractor related data. TBD.

    downloader:
        Dictionary of downloader related data. May contain following data:
            current_fragment:
                Dictionary with current (being downloaded) fragment data:
                index:  0-based index of current fragment among all fragments
            fragment_count:
                Total count of fragments

    This feature is experimental and file format may change in future.
    c                 C   s   |  d | ||||S )Nz{yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. Use yt_dlp.downloader.FileDownloader.report_retry instead)Zdeprecation_warningreport_retry)r   err
frag_indexcountretriesr   r   r   report_retry_fragment>   s    
z FragmentFD.report_retry_fragmentNc                 C   s2   |rd| dnd}|  d| d|dd d S )N ; z
[download]z Skipping fragment dz ...)r   )r   r   r   r   r   r   report_skip_fragmentC   s    zFragmentFD.report_skip_fragmentc                 C   s   | d}|rt|d |S |S )Nhttp_headers)getr   )r   	info_dicturlheadersr   r   r   _prepare_urlG   s    
zFragmentFD._prepare_urlc                 C   s   |  | | || d S r   )_prepare_frag_download_start_frag_download)r   ctxr(   r   r   r    _prepare_and_start_frag_downloadK   s    
z+FragmentFD._prepare_and_start_frag_downloadc                 C   s&   |d duo$|d dko$| j d S )NliveTtmpfilename-Z_no_ytdl_file)paramsr'   )r   r.   r   r   r   Z__do_ytdl_fileO   s    zFragmentFD.__do_ytdl_filec                 C   s   d|vsJ |  | |d d\}}zjzBt| }|d d d |d< d|d v rh|d d |d< W n ty   d	|d< Y n0 W |  n
|  0 d S )
Nytdl_corruptfilenamer
downloadercurrent_fragmentindexfragment_indexextra_stateT)sanitize_openytdl_filenamejsonloadsread	Exceptionclose)r   r.   stream_Z	ytdl_datar   r   r   _read_ytdl_fileR   s    zFragmentFD._read_ytdl_filec                 C   s   |  | |d d\}}z^dd|d ii}d|v r@|d |d< |dd urZ|d |d< |td|i W |  n
|  0 d S )	Nr5   wr8   r9   r:   r;   fragment_countr7   )r<   r=   r'   writer>   dumpsrB   )r   r.   Zfrag_index_streamrD   r7   r   r   r   _write_ytdl_file_   s    zFragmentFD._write_ytdl_filec                 C   s   d|d |d f }||p"| d|| dd}d}|d j d	d
rX| | |}| |d< |d< |d ||\}	}
|	sdS | dr| d|d< ||d< d
S )Nz	%s-Frag%dr1   r:   r&   ctx_id)r)   r&   request_datarK   r   dl
continuedlTfrag_resume_lenFfiletimefragment_filetimefragment_filename_sanitized)r'   r3   filesize_or_none	temp_namedownload)r   r.   Zfrag_urlr(   r*   rL   Zfragment_filenamefragment_info_dictrO   successrD   r   r   r   _download_fragmento   s"    
zFragmentFD._download_fragmentc                 C   sh   | dsd S z| |d d\}}W n$ tyJ   | drDY d S  Y n0 ||d< | }|  |S )NrR   rbr0   )r'   r<   FileNotFoundErrorr@   rB   )r   r.   ZdownZfrag_sanitizedfrag_contentr   r   r   _read_fragment   s    

zFragmentFD._read_fragmentc              
   C   s   zT|d  | |d   W | |r2| | | jddsN| |d  |d= n8| |rj| | | jdds| |d  |d= 0 d S )Ndest_streamZkeep_fragmentsFrR   )rH   flush_FragmentFD__do_ytdl_filerJ   r3   r'   
try_remove)r   r.   r[   r   r   r   _append_fragment   s    



zFragmentFD._append_fragmentc              	   C   s  | dds6d|d  }|dd}|r:|d| 7 }nd}| d	| j d
|  | |d  t| ji | jdddddd}| |d }d}| 	|}|dkrd}|
|dd | |rtj| |d }| jdd}	|	rv|rv| | |ddu }
|d dko"|dk}|
s0|r|
r:dnd}| | d d |d< }d|v rj|d= | | n>|	s|r| | d |d< }| | |d dksJ | ||\}}|
||||d d S )Nr0   F%dtotal_fragsad_fragsr    (not including %d ad)unknown (live)[] Total fragments: r5   T)Z
noprogresstestZsleep_intervalZmax_sleep_intervalZsleep_interval_subtitleswbabr1   r:   rN   r4   r:   z.ytdl file is corruptz2Inconsistent state of incomplete fragment downloadz#. Restarting from the beginning ...)rM   r]   r1   complete_frags_downloaded_bytes)
setdefaultr'   r   FD_NAMEZreport_destinationr   ydlr3   rT   rS   updater_   ospathisfiler=   rE   Zreport_warningrJ   r<   )r   r.   total_frags_strrd   rM   r1   Z	open_mode
resume_lenZytdl_file_existsrN   Z
is_corruptZis_inconsistentmessager]   r   r   r   r,      sr    






z!FragmentFD._prepare_frag_downloadc                    sz    d } d   dd| d  d  d dt  d	< t| fd
d} d |  d	 S )Nrm   rc   rK   downloadingr:   r5   r1   )statusdownloaded_bytesr:   rG   r5   r1   startedc                    sR  | d dvrd S s*  dr* d d< d urD|  dkrDd S   dd<   dd< jd< |  dpvd	}| d
i | d<  d sΈ d | d d   }|_|  d jd< n|  d | d dkrd  d7  < d  d<   j d<  d< jj d<  d< j	jd< 
 d S )Nry   )rx   finishedrG   rK   max_progressprogress_idxelapsedtotal_bytesr   r(   rV   r0   rm   r:   r   rz   Ztotal_bytes_estimater|   speedeta)r'   r   poptotalrq   Zthread_reset
downloadedr   Zsmoothr   _hook_progress)sZfrag_total_bytesZestimated_sizer.   rK   r(   progressr   staterc   r   r   frag_progress_hook   s<    


z;FragmentFD._start_frag_download.<locals>.frag_progress_hookrM   )r'   timer   Zadd_progress_hook)r   r.   r(   rv   r   r   r   r   r-      s    
	'zFragmentFD._start_frag_downloadc                 C   s2  |d    | |r*| | |d  t |d  }|d dk}|rZ| |d }n|d }|s|rx| |d  | d dS |r| |d |d  |d	}| j	d
r|rt
t( t|d t |f W d    n1 s0    Y  | |||d d||d|d|dd| dS )Nr]   r5   r{   r1   r2   rm   zThe downloaded file is emptyFrQ   Z
updatetimer|   rK   r}   r~   )rz   r   r5   ry   r   rK   r}   r~   T)rB   r_   r`   r=   r   rS   report_errorZ
try_renamer'   r3   
contextlibsuppressrA   rr   utimer   )r   r.   r(   r   Zto_filerz   rP   r   r   r   _finish_frag_download  s@    


6	
z FragmentFD._finish_frag_downloadc                 C   s   d|vrd|d< |d sBd|d  }| dd}|rF|d| 7 }nd}| d	| j d
|  | |d }||dd d S )Nr0   Frb   rc   rd   r   re   rf   rg   rh   r5   rl   )r'   r   ro   rT   rq   )r   r.   ru   rd   r1   r   r   r   _prepare_external_frag_downloadA  s    z*FragmentFD._prepare_external_frag_downloadc                    s(   i fdd  fdd}|S )Nc                    s,   |  vr$j |   | <  |  S r   )rp   urlopenr+   r@   )r)   )
_key_cacher(   r   r   r   _get_keyX  s    z&FragmentFD.decrypter.<locals>._get_keyc                    s   |d u rd S |  d}|r&|d dkr*|S | dpBtd| d }| dpb tdp`|d	 |d< j d
drz|S tt||d |S )Ndecrypt_infoZMETHODzAES-128ZIVz>8xqZmedia_sequenceZKEY)Zhls_aesuriURIri   F)r'   structpackr   r3   r   r   )fragmentr[   r   Ziv)r   r(   r   r   r   decrypt_fragment]  s    

z.FragmentFD.decrypter.<locals>.decrypt_fragmentr   )r   r(   r   r   )r   r   r(   r   r   	decrypterU  s    zFragmentFD.decrypterc              
      sv  dg t |dkr*j|d i S jdd}dkrJ tt|d} fdd}G dd	 d	tjj	}t
jd
krdd }ndd } fdd}g }	t|D ]F\}
\}}}|t| }|||
|||||}|	||f qd}|	D ]Z\}}z@z|o||}W n ty:   d d< Y n0 W |jdd n|jdd 0 q d sr|srt|S )z
        @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
                all args must be either tuple or list
        Tr   r   concurrent_fragment_downloads).r   is_livec                    s0   |d< | |d< j |||fi | dS )Nr}   r~   )tpeinterrupt_trigger)download_and_append_fragments)idxr.   	fragmentsr(   r   r   kwargsr}   r   r   r   thread_func}  s    zFFragmentFD.download_and_append_fragments_multiple.<locals>.thread_funcc                   @   s   e Zd Zdd ZdS )z?FragmentFD.download_and_append_fragments_multiple.<locals>.FTPEc                 S   s   d S r   r   )r   exc_typeexc_valexc_tbr   r   r   __exit__  s    zHFragmentFD.download_and_append_fragments_multiple.<locals>.FTPE.__exit__N)r   r   r   r   r   r   r   r   FTPE  s   r   ntc                 S   s@   z|  dW S  ty     Y q  tjjy8   Y q Y q 0 q d S )Ng?)resultKeyboardInterrupt
concurrentfuturesTimeoutErrorfuturer   r   r   future_result  s    zHFragmentFD.download_and_append_fragments_multiple.<locals>.future_resultc                 S   s   |   S r   )r   r   r   r   r   r     s    c                 3   s    | D ]} d s q|V  qd S )Nr   r   )Zfgf)r   r   r   interrupt_trigger_iter  s    zQFragmentFD.download_and_append_fragments_multiple.<locals>.interrupt_trigger_iterFwait)lenr   r3   r'   Z_prepare_multiline_statusanyr   r   r   ThreadPoolExecutorrr   name	enumeratemathceilZsubmitappendr   shutdown)r   r   r   max_workersr   r   r   r   r   Zspinsr   r.   r   r(   r   Zjobr   r   r   r   &download_and_append_fragments_multipleo  s:    


	 z1FragmentFD.download_and_append_fragments_multiplec                 C   s   dS )NFr   )r   r   r   r   <lambda>      zFragmentFD.<lambda>c                 C   s   | S r   r   )contentr   r   r   r   r     r   )T)is_fatal	pack_funcfinish_funcr   r   c             	      s  j ddsdd fddfdd}	}
tj d	d
 dd
 }|d
krB fdd}|ptj|}zX|||D ]F\}}} 	||d |	|
|
 | s W W d    dS qW n: ty     jdddd |jdd  Y n0 W d    n1 s60    Y  n~|D ]x}d sZ qz*|  |	|
|
 |d  }W n* ty   drY  q Y n0 |sF dS qF|d ur d |   d    S )NZskip_unavailable_fragmentsTc                 S   s   dS )NTr   )rD   r   r   r   r     r   z:FragmentFD.download_and_append_fragments.<locals>.<lambda>c                    s.  d sd S | d   d< d  d< t d}| d}|r\d|d |d	 d
 f |d< | dpnd
  fdd}tjd|D ]}z6| d d<  | d |dsW  d S W q ttfy } z||_W Y d }~qW Y d }~qd }~0  ty&   r" Y q0 qd S )Nr   r   r:   
last_errorr&   
byte_rangezbytes=%d-%dstartendr   Ranger9   c                    s6   r||kr d    | || |  d< d S )Nr]   r   )rB   r   )r   r   r   )r.   fatalr   r   r   r   error_callback  s    z[FragmentFD.download_and_append_fragments.<locals>.download_fragment.<locals>.error_callbackZfragment_retriesrG   r)   rL   )	r   r'   r   r3   rX   r	   r
   errorr   )r   r.   r*   r   r   retryr   )r(   r   r   r   )r.   r   r   r   download_fragment  s.    
 zCFragmentFD.download_and_append_fragments.<locals>.download_fragmentc                    sX   | r || | n< |d s2|d n"|d   d| d dS dS )Nr   zfragment not foundr]   z	fragment z not found, unable to continueFT)ra   r%   rB   r   )r[   r   r.   )r   r   r   r   r   append_fragment  s    zAFragmentFD.download_and_append_fragments.<locals>.append_fragmentr   r   r}   c                    s&      }| | | | d |dfS )Nr   rR   )copyr'   )r   Zctx_copy)r.   r   r   r   rX     s    
zDFragmentFD.download_and_append_fragments.<locals>._download_fragment)rR   r:   Fz;Interrupted by user. Waiting for all threads to shutdown...)Zis_errortbr   r   r   r   r]   )r3   r'   r   r   r   r   r   r   maprq   r\   r   Z_finish_multiline_statusr   r   rH   r^   r   )r   r.   r   r(   r   r   r   r   r   r   r   r   rX   poolr   r   Zfrag_filenamer   r   )r.   r   r(   r   r   r   r   r   r     sX    !

*



z(FragmentFD.download_and_append_fragments)N)NN)r   r   r   __doc__r    r%   r+   r/   r_   rE   rJ   rX   r\   ra   r,   r-   r   r   r   r   r   r   r   r   r   r      s*   #

D<$Ar   )concurrent.futuresr   r   r>   r   rr   r   r   commonr   httpr   Zaesr   r   Z
networkingr   Znetworking.exceptionsr	   r
   utilsr   r   r   Zutils.networkingr   Zutils.progressr   r   r   r   r   r   r   <module>   s    