o
     JAfAe                     @   sR  	 d Z dZddlmZ ddlmZmZ ddlZddl	Z	ddl
mZ ddlZddlZddlZddlT ddlmZ ed	ejZed
ejZddlZejg d Z[G dd deddZdd Zdd Zdd ZdZdZeede  ee ZdZdZ dZ!dZ"dZ#ee e!e"e#Z$dZ%eddee% Z&d e% Z'ee&e'Z(ed!e(d" Z)ee)e(e$Z*d#d$ Z+e	j,d%d& Z-ee+  Z.d'Z/d(Z0d)Z1d*Z2ee.d+ e.d, Z3ee.d- e.d. Z4ee5ej6e7ed/d0 Z8ed1e8Z9ee*e9e4eZ:ee: Z;ee.d2 ed3d e.d4 ed5d Z<ed6ee3Z=eee=e*e9e<e Z>i Z?e+ D ]Z@e/e?e@d3 < e0e?e@d5 < e1e?e@d+ < e2e?e@d, < qeA ZBeA ZCe+ D ]%ZDeDd5 eDd3 fD ]ZEeBFeE qHeDd, eDd+ fD ]ZEeCFeE qYq>d7ZGG d8d9 d9eHZIG d:d; d;eHZJG d<d= d=ZKd>d? ZLd@dA ZMdBdC ZNdDdE ZdFdG ZOdHdI ZPdJdK ZQdLdM ZReSdNkreR  dS dS )OzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord    )open)lookupBOM_UTF8N)TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding
untokenize	TokenInfoc                   @   s    e Zd Zdd Zedd ZdS )r   c                 C   s$   d| j t| j  f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r   tok_name_replace)selfannotated_type r   /usr/lib/python3.10/tokenize.py__repr__/   s   
zTokenInfo.__repr__c                 C   s$   | j tkr| jtv rt| j S | j S N)r   OPstringr   r   r   r   r   
exact_type4   s   
zTokenInfo.exact_typeN)__name__
__module____qualname__r   propertyr   r   r   r   r   r   .   s    r   ztype string start end linec                  G   s   dd |  d S )N(|))joinchoicesr   r   r   group;   s    r$   c                  G      t |  d S )Nr   r$   r"   r   r   r   any<       r'   c                  G   r%   )N?r&   r"   r   r   r   maybe=   r(   r*   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c                  C   sV   g d} dh}| D ]}t |D ]}t jdd |D  D ]
}|d| qqq	|S )N)brufbrfr c                 S   s   g | ]}||  fqS r   )upper).0cr   r   r   
<listcomp>_   s    z(_all_string_prefixes.<locals>.<listcomp>)
_itertoolspermutationsproductaddr!   )_valid_string_prefixesresultprefixtr-   r   r   r   _all_string_prefixesT   s   r>   c                 C   s   t | t jS r   )recompileUNICODE)exprr   r   r   _compilec   s   rC   z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z   c                   @      e Zd ZdS )
TokenErrorNr   r   r   r   r   r   r   rI      r(   rI   c                   @   rH   )StopTokenizingNrJ   r   r   r   r   rK      r(   rK   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc                 C   s   g | _ d| _d| _d | _d S )N   r   )tokensprev_rowprev_colencodingr   r   r   r   __init__   s   
zUntokenizer.__init__c                 C   s   |\}}|| j k s|| j kr|| jk rtd||| j | j|| j  }|r1| jd|  d| _|| j }|rB| jd|  d S d S )Nz+start ({},{}) precedes previous end ({},{})\
r    )rO   rP   
ValueErrorformatrN   append)r   startrowcol
row_offset
col_offsetr   r   r   add_whitespace   s   

zUntokenizer.add_whitespacec                 C   s,  t |}g }d}|D ]}t|dkr| ||  nv|\}}}}	}
|tkr)|| _q
|tkr/ na|tkr9|| q
|tkrH|	  |	\| _
| _q
|ttfv rQd}n|rn|rn|d }|d t|krl| j| t|| _d}| | | j| |	\| _
| _|ttfv r|  j
d7  _
d| _q
d| jS )NF   TrM   r   r1   )iterlencompatENCODINGrQ   	ENDMARKERINDENTrW   DEDENTpoprO   rP   NEWLINENLrN   r]   r!   )r   iterableitindents	startliner=   tok_typetokenrX   endlineindentr   r   r   r      sH   


zUntokenizer.untokenizec           
      C   s   g }| j j}|d ttfv }d}t|g|D ]W}|d d \}}	|tkr)|	| _q|tt	fv r3|	d7 }	|t
kr@|r=d|	 }	d}nd}|tkrL||	 q|tkrU|  q|ttfv r^d}n|rj|rj||d  d}||	 qd S )Nr   Fr^   rT   Tr_   )rN   rW   rh   ri   r6   chainrc   rQ   NAMENUMBERSTRINGre   rf   rg   )
r   ro   rj   rl   toks_appendrm   
prevstringtoktoknumtokvalr   r   r   rb      s:   

zUntokenizer.compatN)r   r   r   rR   r]   r   rb   r   r   r   r   rL      s
    %rL   c                 C   s,   	 t  }|| }|jd ur||j}|S r   )rL   r   rQ   encode)rj   utoutr   r   r   r     s   

r   c                 C   sJ   	 | d d   dd}|dks|drdS |dv s!|dr#dS | S )	N   _-utf-8zutf-8-)zlatin-1
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-r   )lowerreplace
startswith)orig_encencr   r   r   _get_normal_name  s   r   c                    s   	 zj jW n ty   d Y nw d d }d}fdd} fdd}| }|tr8d |dd  }d	}|s>|g fS ||}|rI||gfS t|sS||gfS | }|s]||gfS ||}|ri|||gfS |||gfS )
NFr   c                      s   z  W S  t y   Y dS w N    )StopIterationr   readliner   r   read_or_stopA  s
   z%detect_encoding.<locals>.read_or_stopc                    s   z|  d}W n ty   d}d urd|}t|w t|}|s(d S t|d}zt|}W n t	yR   d u rHd| }t|d|}t|w  rp|dkrld u rcd}t|d}t||d	7 }|S )
Nr   z'invalid or missing encoding declarationz{} for {!r}rM   zunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrV   SyntaxError	cookie_rematchr   r$   r   LookupError)rq   line_stringmsgr   rQ   codec)	bom_foundfilenamer   r   find_cookieG  s@   
	
z$detect_encoding.<locals>.find_cookieT   	utf-8-sig)__self__nameAttributeErrorr   r   blank_rer   )r   rQ   defaultr   r   firstsecondr   )r   r   r   r   r
   )  s<   &




r
   c                 C   sP   	 t | d}zt|j\}}|d t||dd}d|_|W S    |   )Nrbr   T)line_bufferingr,   )_builtin_openr
   r   seekr   modeclose)r   bufferrQ   linestextr   r   r   r     s   

r   c                 C   s8   	 t | \}}td}t|t| d|}t|j|S r   )r
   r6   repeatrs   r`   	_tokenize__next__)r   rQ   consumedemptyrl_genr   r   r   r     s
   
r   c                 c   sl   d } }}d}d\}}d }dg}	|d ur%|dkrd}t t|dddV  d}
d}	 z|}
|  }W n ty<   d}Y nw |d urF||}|d
7 }dt|}}|r|sZtd|||}|r|d }}t t||d |  |||f|| V  d\}}d }n|r|dd  dkr|dd  dkrt t	|| ||t|f|V  d}d }q)|| }|| }q)|dkr|s|sn)d}||k r|| dkr|d
7 }n|| dkr|t
 d
 t
 }n
|| dkrd}nn|d
7 }||k s||krn|| dv r8|| dkr$||d  d}t t|||f||t| f|V  |t|7 }t t||d  ||f|t|f|V  q)||	d krU|	| t t|d | |df||f|V  ||	d k r||	vrjtdd|||f|	d d }	t td||f||f|V  ||	d k s\n|std|dfd}||k rtt||}|r|d
\}}||f||f|}}}||krq||| || }}||v s|dkr|dkr|dkrt t||||V  n|dv r|dkrt t||||V  nt t||||V  n|dkrt t||||V  n|tv rItt| }|||}|r<|d}||| }t t||||f|V  n||f}||d  }|}n|tv s`|d d tv s`|d d tv r|d dkr||f}tt|pt|d
 pt|d }||d  d
}}|}nXt t||||V  nI| rt t||||V  n:|dkrd
}n2|d v r|d
7 }n	|d!v r|d
8 }t t||||V  nt t	|| ||f||d
 f|V  |d
7 }||k sq*|
r|
d dvr|
 dst td|d
 t|
f|d
 t|
d
 fdV  |	d
d  D ]}t td|df|dfdV  qt t d|df|dfdV  d S )"Nr   
0123456789)r1   r   r   r   )r   r   r1   r   TrM   zEOF in multi-line stringrS   z\
rT   	z#
#z
r_   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.z...r^   r   
\z([{z)]})!r   rc   r   r   ra   rI   r   rp   rv   
ERRORTOKENtabsizerstripCOMMENTri   rW   re   IndentationErrorrf   rC   PseudoTokenspanru   rh   triple_quotedendpatssingle_quotedgetisidentifierrt   r   stripr   rd   )r   rQ   lnumparenlev	continuednumcharscontstrneedcontcontlinerl   	last_linerq   posmaxstrstartendprogendmatchrp   columncomment_tokenpseudomatchrX   sposeposro   initialrr   r   r   r   r     s*  



$



"


















 $#.r   c                 C   s   	 t | d S r   )r   r   r   r   r   r	   f  s   
r	   c               
      s2  dd l } dd  d fdd	}| jdd}|jdd	d
dd |jdddddd | }zQ|jrP|j}t|d}tt|j}W d    n1 sJw   Y  n	d}t	t
jjd }|D ]}|j}|jrf|j}d|j|j  }	td|	t| |jf  q[W d S  ty }
 z|
jd dd \}}||
jd |||f W Y d }
~
d S d }
~
w ty }
 z|
jd \}}||
jd |||f W Y d }
~
d S d }
~
w ty }
 z||
| W Y d }
~
d S d }
~
w ty }
 z||
 W Y d }
~
d S d }
~
w ty   td Y d S  ty }
 z d|
   d }
~
ww )Nr   c                 S   s   t j|  t jd d S )Nr   )sysstderrwrite)messager   r   r   perrorr  s   zmain.<locals>.perrorc                    sR   |r|f| | f } d|  n|r d|| f  n d|   t d d S )Nz%s:%d:%d: error: %sz%s: error: %sz	error: %srM   )r   exit)r   r   locationargsr   r   r   errorv  s   zmain.<locals>.errorzpython -m tokenize)progr   r)   zfilename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r   actionr   r   z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrM   r   zinterrupted
zunexpected error: %s)NN)argparseArgumentParseradd_argument
parse_argsr   r   listr   r   r   r   stdinr   r   r   rX   rp   printr   r   r   r   rI   r   OSErrorKeyboardInterrupt	Exception)r   r   parserr   r   r.   rN   ro   
token_typetoken_rangeerrrq   r   r   r   r   mainn  sf   $$r   __main__)T
__author____credits__builtinsr   r   codecsr   r   collections	functoolsior   	itertoolsr6   r?   r   ro   r   r@   ASCIIr   r   __all__
namedtupler   r$   r'   r*   
WhitespaceCommentIgnoreName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberr>   	lru_cacherC   StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStrPseudoExtrasr   r   _prefixsetr   r   r=   r-   r9   r   r   rI   rK   rL   r   r   r
   r   r   r	   r   r   r   r   r   r   <module>   s    





_] 8
=
