관리-도구
편집 파일: tokenize.cpython-310.opt-2.pyc
o 6��fAe � @ sR d Z dZddlmZ ddlmZmZ ddlZddl Z ddl m Z ddl Z ddlZddlZddlT ddlmZ e�d ej�Ze�d ej�ZddlZejg d � Z[G d d � d e�d d��Zdd� Zdd� Zdd� ZdZ dZ e ede � ee � Z dZ dZ dZ!dZ"d Z#ee e!e"e#�Z$d Z%ed d �ee%� Z&d e% Z'ee&e'�Z(ed!e(d" �Z)ee)e(e$�Z*d#d$� Z+e j,d%d&� �Z-ee+� � Z.d'Z/d(Z0d)Z1d*Z2ee.d+ e.d, �Z3ee.d- e.d. �Z4ee5ej6e7ed/d0��� Z8ed1e8�Z9ee*e9e4e �Z:e e: Z;ee.d2 ed3d� e.d4 ed5d� �Z<ed6e e3�Z=e ee=e*e9e<e � Z>i Z?e+� D ]Z@e/e?e@d3 < e0e?e@d5 < e1e?e@d+ < e2e?e@d, < �qeA� ZBeA� ZCe+� D ]%ZDeDd5 eDd3 fD ]ZEeB�FeE� �qHeDd, eDd+ fD ]ZEeC�FeE� �qY�q>d7ZGG d8d9� d9eH�ZIG d:d;� d;eH�ZJG d<d=� d=�ZKd>d?� ZLd@dA� ZMdBdC� ZNdDdE� ZdFdG� ZOdHdI� ZPdJdK� ZQdLdM� ZReSdNk�r�eR� dS dS )OzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)� TextIOWrapper)�*)�EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))�tokenize�generate_tokens�detect_encoding� untokenize� TokenInfoc @ s e Zd Zdd� Zedd� �ZdS )r c C s$ d| j t| j f }d| j|d� S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r �tok_name�_replace)�self�annotated_type� r �//opt/alt/python310/lib64/python3.10/tokenize.py�__repr__/ s �zTokenInfo.__repr__c C s$ | j tkr| jtv rt| j S | j S �N)r �OP�stringr �r r r r � exact_type4 s zTokenInfo.exact_typeN)�__name__� __module__� __qualname__r �propertyr r r r r r . s r ztype string start end linec G s dd� | � d S )N�(�|�))�join��choicesr r r �group; s r$ c G � t | � d S )Nr �r$ r" r r r �any< � r' c G r% )N�?r& r" r r r �maybe= r( r* z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sV g d�} dh}| D ] }t �|�D ]}t jdd� |D �� D ] }|�d�|�� q qq |S )N)�b�r�u�f�br�fr� c S s g | ]}||� � f�qS r )�upper)�.0�cr r r � <listcomp>_ s z(_all_string_prefixes.<locals>.<listcomp>)� _itertools� permutations�product�addr! )�_valid_string_prefixes�result�prefix�tr- r r r �_all_string_prefixesT s ��r> c C s t �| t j�S r )�re�compile�UNICODE)�exprr r r �_compilec s rC z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)�reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z \\\r?\n|\Z� c @ � e Zd ZdS )� TokenErrorN�r r r r r r r rI � r( rI c @ rH )�StopTokenizingNrJ r r r r rK � r( rK c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � Untokenizerc C s g | _ d| _d| _d | _d S )N� r )�tokens�prev_row�prev_col�encodingr r r r �__init__� s zUntokenizer.__init__c C s� |\}}|| j k s|| j kr || jk r td�||| j | j���|| j }|r1| j�d| � d| _|| j }|rB| j�d| � d S d S )Nz+start ({},{}) precedes previous end ({},{})�\ r � )rO rP � ValueError�formatrN �append)r �start�row�col� row_offset� col_offsetr r r �add_whitespace� s � �zUntokenizer.add_whitespacec C s, t |�}g }d}|D ]�}t|�dkr| �||� nv|\}}}} } |tkr)|| _q |tkr/ na|tkr9|�|� q |tkrH|� � | \| _ | _ q |t t fv rQd}n |rn|rn|d } |d t| �krl| j�| � t| �| _ d}| �|� | j�|� | \| _ | _ |t t fv r�| j d7 _ d| _ q d�| j�S )NF� T���rM r r1 )�iter�len�compat�ENCODINGrQ � ENDMARKER�INDENTrW �DEDENT�poprO rP �NEWLINE�NLrN r] r! ) r �iterable�it�indents� startliner= �tok_type�tokenrX �end�line�indentr r r r � sH � zUntokenizer.untokenizec C s� g }| j j}|d ttfv }d}t�|g|�D ]W}|d d� \}} |tkr)| | _q|tt fv r3| d7 } |t kr@|r=d| } d}nd}|t krL|�| � q|t krU|� � q|ttfv r^d}n |rj|rj||d � d}|| � qd S )Nr Fr^ rT Tr_ )rN rW rh ri r6 �chainrc rQ �NAME�NUMBER�STRINGre rf rg ) r ro rj rl � toks_appendrm � prevstring�tok�toknum�tokvalr r r rb � s: �zUntokenizer.compatN)r r r rR r] r rb r r r r rL � s %rL c C s, t � }|�| �}|jd ur|�|j�}|S r )rL r rQ �encode)rj �ut�outr r r r s r c C sJ | d d� � � �dd�}|dks|�d�rdS |dv s!|�d�r#dS | S ) N� �_�-�utf-8zutf-8-)zlatin-1� iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r� )�lower�replace� startswith)�orig_enc�encr r r �_get_normal_name s �r� c s� z�j j�W n ty d �Y nw d� d }d}�fdd�}� �fdd�}|� }|�t�r8d� |dd � }d }|s>|g fS ||�}|rI||gfS t�|�sS||gfS |� }|s]||gfS ||�}|ri|||gfS |||gfS ) NFr� c s z� � W S t y Y dS w �N� )� StopIterationr ��readliner r � read_or_stopA s �z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n ty d}�d urd�|��}t|��w t�|�}|s(d S t|�d��}zt|�}W n t yR �d u rHd| }t|��d��|�}t|��w � rp|dkrl�d u rcd}t|��d���}t|��|d 7 }|S ) Nr� z'invalid or missing encoding declarationz {} for {!r}rM zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) �decode�UnicodeDecodeErrorrV � SyntaxError� cookie_re�matchr� r$ r � LookupError)rq � line_string�msgr� rQ �codec)� bom_found�filenamer r � find_cookieG s@ � ��� �z$detect_encoding.<locals>.find_cookieT� � utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� rQ �defaultr� r� �first�secondr )r� r� r� r r ) s< � & r c C sP t | d�}zt|j�\}}|�d� t||dd�}d|_|W S |�� � )N�rbr T)�line_bufferingr, )� _builtin_openr r� �seekr �mode�close)r� �bufferrQ �lines�textr r r r � s r c C s8 t | �\}}t�d�}t�|t| d�|�}t|j|�S r� )r r6 �repeatrs r` � _tokenize�__next__)r� rQ �consumed�empty�rl_genr r r r � s r c c sl � d } }}d}d\}}d }dg} |d ur%|dkr d}t t|ddd�V d} d} z| } | � } W n ty< d} Y nw |d urF| �|�} |d 7 }dt| �} } |r�| sZtd |��|�| �}|r�|�d� } }t t|| d |� |||f|| �V d\}}d }�n|r�| d d � d kr�| dd � dkr�t t || ||t| �f|�V d}d }q)|| }|| }q)|dk�r�|�s�| s��n)d}| | k r�| | dkr�|d 7 }n| | dkr�|t d t }n | | dkr�d}nn| d 7 } | | k s�| | kr��n�| | dv �r8| | dk�r$| | d � � d�}t t ||| f|| t|� f| �V | t|�7 } t t | | d � || f|t| �f| �V q)|| d k�rU| �|� t t| d | � |df|| f| �V || d k �r�|| v�rjtdd|| | f��| d d� } t td|| f|| f| �V || d k �s\n | �s�td|df��d}| | k �r�tt��| | �}|�r�|�d �\}}||f||f|}}} ||k�r��q�| ||� | | }}||v �s�|dk�r�|dk�r�|dk�r�t t|||| �V n�|dv �r|dk�r�t t |||| �V n�t t|||| �V n�|dk�rt t |||| �V n�|tv �rItt| �}|�| | �}|�r<|�d�} | || � }t t|||| f| �V n�||f}| |d � }| }n�|tv �s`|d d � tv �s`|d d � tv �r�|d d k�r�||f}tt�|��p�t�|d ��p�t�|d ��}| |d � d }}| }nXt t|||| �V nI|�� �r�t t |||| �V n:|d k�r�d }n2|d v �r�|d 7 }n |d!v �r�|d 8 }t t |||| �V nt t | | || f|| d f| �V | d 7 } | | k �s�q*| �r| d dv�r| � � � d��st td|d t| �f|d t| �d fd�V | d d � D ]}t td|df|dfd�V �qt t d|df|dfd�V d S )"Nr � 0123456789)r1 r r� r� )r r r1 r� TrM zEOF in multi-line string���rS ���z\ rT � � z# �#z r_ z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statement�.z...r^ r� � �\z([{z)]})!r rc r� r� ra rI r� rp rv � ERRORTOKEN�tabsize�rstrip�COMMENTri rW re �IndentationErrorrf rC � PseudoToken�spanru rh � triple_quoted�endpats� single_quoted�get� isidentifierrt r �stripr� rd )r� rQ �lnum�parenlev� continued�numchars�contstr�needcont�contlinerl � last_linerq �pos�max�strstart�endprog�endmatchrp �column� comment_token� pseudomatchrX �spos�eposro �initialrr r r r r� � s* � � �$ � � � � " � �� � � � �� $#. r� c C s t | d �S r )r� r� r r r r f s r c s2 dd l } dd� � d� fdd� }| jdd�}|jdd d d d � |jd ddddd� |�� }zQ|jrP|j}t|d��}tt|j��}W d � n1 sJw Y n d}t t j jd �}|D ] }|j }|j rf|j}d|j|j } td| t| |jf � q[W d S ty� } z | jd dd� \} } || jd || | f� W Y d } ~ d S d } ~ w ty� } z| jd \} } || jd || | f� W Y d } ~ d S d } ~ w ty� } z || |� W Y d } ~ d S d } ~ w ty� } z || � W Y d } ~ d S d } ~ w t�y td� Y d S t�y } z� d| � � d } ~ ww ) Nr c S s t j�| � t j�d� d S )Nr� )�sys�stderr�write)�messager r r �perrorr s zmain.<locals>.perrorc sR |r|f| | f }� d| � n|r � d|| f � n� d| � t �d� d S )Nz%s:%d:%d: error: %sz %s: error: %sz error: %srM )r� �exit)r� r� �location�args�r� r r �errorv s zmain.<locals>.errorzpython -m tokenize)�progr� r) z filename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact� store_truez(display token names using the exact type)r� �actionr� r� z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rrM r� z interrupted zunexpected error: %s)NN)�argparse�ArgumentParser� add_argument� parse_argsr� r� �listr r� r� r� �stdinr r� r rX rp �printr r r� r� rI r� �OSError�KeyboardInterrupt� Exception) r� r� �parserr� r� r. rN ro � token_type� token_range�errrq r� r r� r �mainn sf � � ����$�$��� ��r� �__main__)T� __author__� __credits__�builtinsr r� �codecsr r � collections� functools�ior � itertoolsr6 r? r� ro r r@ �ASCIIr� r� �__all__� namedtupler r$ r' r* � Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�Numberr> � lru_cacherC � StringPrefix�Single�Double�Single3�Double3�Triple�String�map�escape�sorted�Special�Funny� PlainToken�Token�ContStr� PseudoExtrasr� r� �_prefix�setr� r� r= r- r9 r� r� rI rK rL r r� r r r� r r� r r r r r �<module> s� �� � ��� �_ ] 8 = �