관리-도구
편집 파일: tokenize.cpython-37.opt-2.pyc
B YI~d_R � @ s� d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd � e e �D �d d d g Z [ ye W n e k rz eZ Y nX d d� Zdd� Zdd� Zdd� ZdZdZeede � ee� ZdZdZdZdZed d �Zeeeee�Zd Z ed d �ee � Z d!e Z ee e �Z ed"e d# �Z ee e e�Z!d$Z"d%Z#d&Z$d'Z%d(Z&ee&d) e&d* �Z'ee&d+ e&d, �Z(ed-d.d/d0d1d2d3d4d5� Z)d6Z*ed7d8d9�Z+ee)e*e+�Z,ee!e,e(e�Z-ee- Z.ee&d: ed;d� e&d< ed=d� �Z/edee'�Z0eee0e!e,e/e� Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed>d?d@dA�ed>d?dBdC�B dDdEdFdGdHdIhB Z8e�3e"�e�3e#�e6e7dJ�dKdL� e8D �dMdL� e8D �dNdL� e8D ��Z9d)d*hdOdP� e8D �B dQdP� e8D �B Z:d;d=hdRdP� e8D �B dSdP� e8D �B Z;dTZ<G dUdV� dVe=�Z>G dWdX� dXe=�Z?dYdZ� Z@e@fd[d �ZAd\d]� ZBG d^d_� d_�ZCe�3d`ejD�ZEe�3daejD�ZFdbdc� ZGddde� ZHdfd � ZIdgd � ZJeKdhk�r�ddlLZLeMeLjN�dk�r�eAeOeLjNd �jP� n eAeLjQjP� dS )izKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s g | ]}|d dkr|�qS )r �_� )�.0�xr r �;/opt/alt/python37/lib64/python3.7/lib2to3/pgen2/tokenize.py� <listcomp>% s r �tokenize�generate_tokens� untokenizec G s dd� | � d S )N�(�|�))�join)�choicesr r r �group0 � r c G s t | � d S )Nr )r )r r r r �any1 r r c G s t | � d S )N�?)r )r r r r �maybe2 r r c s t � fdd�� D ��S )Nc 3 s4 | ],}� d D ] }|� � |� � kr|| V qqdS ))� N)�casefold)r r �y)�lr r � <genexpr>5 s z _combinations.<locals>.<genexpr>)�set)r r )r r � _combinations3 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B�u�U�urZuRZUrZUR)r" r# z'''z"""c C s i | ]}t |� d ��qS )z''')� single3prog)r �prefixr r r � <dictcomp>y s r/ c C s i | ]}t |� d ��qS )z""")� double3prog)r r. r r r r/ z s c C s i | ] }d |�qS )Nr )r r. r r r r/ { s c C s h | ]}|� d ��qS )z'''r )r r. r r r � <setcomp> s r1 c C s h | ]}|� d ��qS )z"""r )r r. r r r r1 � s c C s h | ]}|� d ��qS )r" r )r r. r r r r1 � s c C s h | ]}|� d ��qS )r# r )r r. r r r r1 � s � c @ s e Zd ZdS )� TokenErrorN)�__name__� __module__� __qualname__r r r r r3 � s r3 c @ s e Zd ZdS )�StopTokenizingN)r4 r5 r6 r r r r r7 � s r7 c C s4 |\}}|\}}t d||||t| t|�f � d S )Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r � printtoken� s r= c C s( yt | |� W n tk r" Y nX d S )N)� tokenize_loopr7 )�readline� tokeneaterr r r r � s c C s xt | �D ] }||� q W d S )N)r )r? r@ Z token_infor r r r> � s r> c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � Untokenizerc C s g | _ d| _d| _d S )Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s* |\}}|| j }|r&| j�d| � d S )N� )rD rB �append)rE �start�row�col� col_offsetr r r �add_whitespace� s zUntokenizer.add_whitespacec C s� xv|D ]n}t |�dkr$| �||� P |\}}}}}| �|� | j�|� |\| _| _|ttfkr| jd7 _d| _qW d� | j�S )N� r r r ) �len�compatrM rB rH rC rD �NEWLINE�NLr )rE �iterable�t�tok_typer rI �endr< r r r r � s zUntokenizer.untokenizec C s� d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x�|D ]�}|d d� \}}|ttttfkrn|d7 }|tkr�|�|� qBn>|t kr�|� � qBn*|ttfkr�d}n|r�|r�||d � d}||� qBW d S )NFrG TrN ���) rB rH �NAME�NUMBERrQ rR �ASYNC�AWAIT�INDENT�DEDENT�pop) rE r rS � startline�indents� toks_append�toknum�tokval�tokr r r rP � s0 zUntokenizer.compatN)r4 r5 r6 rF rM r rP r r r r rA � s rA z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d� � � �dd�}|dks*|�d�r.dS |dks@|�d�rDdS | S ) N� r �-zutf-8zutf-8-)zlatin-1z iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-z iso-8859-1)�lower�replace� startswith)�orig_enc�encr r r �_get_normal_name� s rl c s� d� d }d}�fdd�}� fdd�}|� }|� t�rHd� |dd � }d }|sT|g fS ||�}|rj||gfS t�|�s~||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) NFzutf-8c s" y� � S t k r t� S X d S )N)� StopIteration�bytesr )r? r r � read_or_stop s z%detect_encoding.<locals>.read_or_stopc s� y| � d�}W n tk r" d S X t�|�}|s6d S t|�d��}y t|�}W n tk rp td| ��Y nX � r�|j dkr�td��|d7 }|S )N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig) �decode�UnicodeDecodeError� cookie_re�matchrl r r � LookupError� SyntaxError�name)r< � line_stringrt �encoding�codec)� bom_foundr r � find_cookie s" z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)ri r �blank_rert )r? ry �defaultro r| �first�secondr )r{ r? r �detect_encoding� s0 r� c C s t � }|�| �S )N)rA r )rS �utr r r r : s c c s� d } }}d\}}d }dg}d }d} d} d} �x�y | � } W n t k rT d} Y nX |d }dt| � } }|�r6| s�td|��|�| �}|r�|�d� } }t|| d |� |||f|| fV d\}}d }nd|�r | dd � dk�r | d d � d k�r t|| ||t| �f|fV d}d }q2n|| }|| }q2�nD|dk�rb|�sb| �sNP d}xf| |k �r�| | d k�rv|d }n6| | d k�r�|t d t }n| | d k�r�d}nP | d } �qTW | |k�r�P |�r�|V d }| | dk�r�| | dk�rV| | d � �d�}| t|� }t ||| f|| t|� f| fV t | |d � ||f|t| �f| fV q2t t f| | dk | | d � || f|t| �f| fV q2||d k�r�|� |� t | d | � |df|| f| fV xt||d k �r8||k�r�t dd|| | f��|d d� }| �r | |d k�r d} d} d} td|| f|| f| fV �q�W | �rz| �rz| |d k�rzd} d} d} n| �svtd|df��d}�x�| |k �r&t�| | �}|�r�|�d�\}}||f||f| }}} | ||� | | }}|tjk�s�|dk�r |dk�r t|||| fV �q"|dk�rVt} |dk�r(t } n | �r2d} |�rB|V d }| |||| fV �q"|dk�r�|�rp|V d }t |||| fV �q"|tk�rt| }|�| | �}|�r�|�d�} | || � }|�r�|V d }t|||| f| fV n||f}| |d � }| }P �q"|tk�s0|d d� tk�s0|d d� tk�r�|d dk�r�||f}t| �pht|d �pht|d }| |d � d }}| }P n |�r�|V d }t|||| fV �q"|�� �r~|dk�r�| �r�|dk�r�tnt|||| fV �q~t|||| f} |dk�r |�s | }�q~|d k�rf|�rf|d tk�rf|d dk�rfd} |d } t|d |d |d |d fV d }|�rv|V d }| V nz|d k�r�|�r�|V d }t |||| f| fV d}nF|d k�r�|d }n|d k�r�|d }|�r�|V d }t |||| fV n(t| | || f|| d f| fV | d } �q~W q2W |�r<|V d }x.|dd � D ] } td|df|dfdfV �qJW t d|df|dfdfV d S )!Nr )r r Fr r zEOF in multi-line string���z\ ���z\ rG � � z# �#z rW z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statement�.TrN r} � )�async�awaitr� �def� �\z([{z)]}) rm rO r3 rt rV �STRING� ERRORTOKEN�tabsize�rstrip�COMMENTrR rH r\ �IndentationErrorr] � pseudoprog�span�string�digitsrY rQ � triple_quoted�endprogs� single_quoted� isidentifierrZ r[ rX �OP� ENDMARKER) r? �lnum�parenlev� continued�contstr�needcont�contliner` ZstashedZ async_defZasync_def_indentZ async_def_nlr< �pos�max�strstart�endprog�endmatchrV �column� comment_tokenZnl_pos� pseudomatchrI �spos�eposr �initial�newlinerd �indentr r r r O sn * �__main__)R� __author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenr r �dir�__all__rn � NameError�strr r r r � Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�Number�Single�Double�Single3�Double3Z _litprefix�Triple�String�Operator�Bracket�Special�Funny� PlainToken�Token�ContStr� PseudoExtras� PseudoToken�map�compileZ tokenprogr� r- r0 Z _strprefixesr� r� r� r� � Exceptionr3 r7 r= r r> rA �ASCIIrs r~ rl r� r r r4 �sysrO �argv�openr? �stdinr r r r �<module> s� 8 I b