Current File : //opt/alt/python35/lib64/python3.5/lib2to3/pgen2/__pycache__/tokenize.cpython-35.opt-2.pyc
���],U � T @ s d Z d Z d d l Z d d l Z d d l m Z m Z d d l Td d l m Z d d � e
e � D� d
d d g Z [ y e Wn e
k
r� e Z Yn Xd
d � Z d d � Z d d � Z d Z d Z e e d e � e e � Z d Z d Z d Z d Z d Z e e e e e � Z d Z e d d � e e � Z d e Z e e e � Z e d e d � Z e e e e � Z d! Z! d" Z" d# Z# d$ Z$ e d% d& � Z% e d' d( � Z& e d) d* d+ d, d- d. d/ d0 d1 � Z' d2 Z( e d3 d4 � Z) e e' e( e) � Z* e e e* e& e � Z+ e e+ Z, e d5 e d6 d � d7 e d8 d � � Z- e d e e% � Z. e e e. e e* e- e � Z/ e0 e1 e j2 e, e/ e# e$ f � � \ Z3 Z4 Z5 Z6 d6 e j2 e! � d8 e j2 e" � d9 e5 d: e6 d; e5 d< e6 d= e5 d> e6 d? e5 d@ e6 dA e5 dB e6 dC e5 dD e6 dE e5 dF e6 dG e5 dH e6 dI e5 dJ e6 dK e5 dL e6 dM e5 dN e6 dO e5 dP e6 dQ e5 dR e6 dS e5 dT e6 dU e5 dV e6 dW d dX d dY d dZ d d[ d d\ d i& Z7 i Z8 x d� D] Z9 e9 e8 e9 <q�Wi Z: x d� D] Z9 e9 e: e9 <q�Wdy Z; Gdz d{ � d{ e< � Z= Gd| d} � d} e< � Z> d~ d � Z? e? d� d
� Z@ d� d� � ZA Gd� d� � d� � ZB e j2 d� e jC � ZD e j2 d� e jC � ZE d� d� � ZF d� d� � ZG d� d � ZH d� d � ZI eJ d� k rd d lK ZK eL eK jM � d k r�e@ eN eK jM d � jO � n e@ eK jP jO � d S)�zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro� N)�BOM_UTF8�lookup)�*� )�tokenc C s& g | ] } | d d k r | � q S)r �_� )�.0�xr r �-/opt/alt/python35/lib64/python3.5/tokenize.py�
<listcomp>% s r �tokenize�generate_tokens�
untokenizec G s d d j | � d S)N�(�|�))�join)�choicesr r r �group0 s r c G s t | � d S)Nr )r )r r r r �any1 s r c G s t | � d S)N�?)r )r r r r �maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z
0[bB][01]*z0[xX][\da-fA-F]*[lL]?z0[oO]?[0-7]*[lL]?z
[1-9]\d*[lL]?z [eE][-+]?\d+z\d+\.\d*z\.\d+z\d+z\d+[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z[ubUB]?[rR]?'''z[ubUB]?[rR]?"""z&[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'z&[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz[:;.,`@]z'[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�'''�"""�r'''�r"""�u'''�u"""�b'''�b"""�ur'''�ur"""�br'''�br"""�R'''�R"""�U'''�U"""�B'''�B"""�uR'''�uR"""�Ur'''�Ur"""�UR'''�UR"""�bR'''�bR"""�Br'''�Br"""�BR'''�BR"""�r�R�u�U�b�B�r'�r"�R'�R"�u'�u"�U'�U"�b'�b"�B'�B"�ur'�ur"�Ur'�Ur"�uR'�uR"�UR'�UR"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"� c @ s e Z d Z d S)�
TokenErrorN)�__name__�
__module__� __qualname__r r r r r] � s r] c @ s e Z d Z d S)�StopTokenizingN)r^ r_ r` r r r r ra � s ra c C sF | \ } } | \ } } t d | | | | t | t | � f � d S)Nz%d,%d-%d,%d: %s %s)�print�tok_name�repr) �typer Zxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolr r r �
printtoken� s rg c C s* y t | | � Wn t k
r% Yn Xd S)N)�
tokenize_loopra )�readline�
tokeneaterr r r r
� s
c C s% x t | � D] } | | � q
Wd S)N)r )ri rj Z
token_infor r r rh � s rh c @ s@ e Z d Z d d � Z d d � Z d d � Z d d � Z d S)
� Untokenizerc C s g | _ d | _ d | _ d S)Nr r )�tokens�prev_row�prev_col)�selfr r r �__init__� s zUntokenizer.__init__c C s7 | \ } } | | j } | r3 | j j d | � d S)N� )rn rl �append)ro �start�row�col�
col_offsetr r r �add_whitespace� s
zUntokenizer.add_whitespacec C s� x� | D]� } t | � d k r0 | j | | � P| \ } } } } } | j | � | j j | � | \ | _ | _ | t t f k r | j d 7_ d | _ q Wd j | j � S)N� r r � )
�len�compatrw rl rr rm rn �NEWLINE�NLr )ro �iterable�t�tok_typer rs �endrf r r r r � s
zUntokenizer.untokenizec C s% d } g } | j j } | \ } } | t t f k r@ | d 7} | t t f k rX d } x� | D]� } | d d � \ } } | t t t t f k r� | d 7} | t k r� | j | � q_ nW | t k r� | j
� q_ n; | t t f k r� d } n | r| r| | d � d } | | � q_ Wd S)NFrq Trx r ���) rl rr �NAME�NUMBERr| r} �ASYNC�AWAIT�INDENT�DEDENT�pop) ro r r~ � startline�indents� toks_append�toknum�tokval�tokr r r r{ � s0
zUntokenizer.compatN)r^ r_ r` rp rw r r{ r r r r rk � s rk z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sd | d d � j � j d d � } | d k s= | j d � rA d S| d k s\ | j d
� r` d S| S)N� r �-zutf-8zutf-8-�latin-1�
iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name� s " r� c s d � d } d } � f d d � } � f d d � } | � } | j t � rj d � | d d � } d } | sz | g f S| | � } | r� | | g f St j | � s� | | g f S| � } | s� | | g f S| | � } | r� | | | g f S| | | g f S)
NFzutf-8c s+ y � � SWn t k
r& t � SYn Xd S)N)�
StopIteration�bytesr )ri r r � read_or_stop s
z%detect_encoding.<locals>.read_or_stopc s� y | j d � } Wn t k
r+ d SYn Xt j | � } | sE d St | j d � � } y t | � } Wn" t k
r� t d | � � Yn X� r� | j d k r� t d � � | d 7} | S)N�asciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError� cookie_re�matchr� r r � LookupError� SyntaxError�name)rf � line_stringr� �encoding�codec)� bom_foundr r � find_cookie s"
z$detect_encoding.<locals>.find_cookieT� z utf-8-sig)r� r �blank_rer� )ri r� �defaultr� r� �first�secondr )r� ri r �detect_encoding� s0
r� c C s t � } | j | � S)N)rk r )r~ �utr r r r F s c ! c sY d } } } t j d d } } d \ } } d } d g } d }
d } d } d }
x�y
| � } Wn t k
r d } Yn X| d } d t | � } } | r�| s� t d | � � | j | � } | r!| j d � } } t | | d | � | | | f | | f Vd \ } } d } qv| r�| d! d � d k r�| d" d � d k r�t | | | | t | � f | f Vd } d } qX qv| | } | | } qX n�| d k rU| rU| s�Pd } xv | | k r;| | d k r�| d } n? | | d
k r| t d t } n | | d k r-d } n P| d } q�W| | k rIP|
rZ|
Vd }
| | d k rG| | d k r | | d � j
d � } | t | � } t | | | f | | t | � f | f Vt | | d � | | f | t | � f | f VqX t t f | | d k | | d � | | f | t | � f | f VqX | | d# k r�| j
| � t | d | � | d f | | f | f Vx� | | d$ k r#| | k r�t d d | | | f � � | d d% � } | r | | d& k r d } d }
d } t d | | f | | f | f Vq�W| rv|
rv| | d' k rvd } d }
d } n! | spt d | d f � � d } xl| | k r�t j | | � } | r�| j d � \ } } | | f | | f | } } } | | | � | | } } | | k s| d k r(| d k r(t | | | | f Vq�| d k r�t } | d k rOt } n | r[d }
|
rl|
Vd }
| | | | | f Vq�| d k r�|
r�|
Vd }
t | | | | f Vq�| t k rXt | } | j | | � } | r2| j d � } | | | � } |
r|
Vd }
t | | | | f | f Vq�| | f } | | d � } | } Pq�| t k s�| d d � t k s�| d d
� t k r| d( d k r�| | f } t | p�t | d p�t | d } | | d � d } } | } Pq�|
r|
Vd }
t | | | | f Vq�| | k r| d) k rb| rb| d k rKt n t | | | | f Vqyt | | | | f } | d k r�|
r�| }
qy| d k r�|
r�|
d t k r�|
d d k r�d } | d* } t |
d |
d |
d
|
d f Vd }
|
r|
Vd }
| Vq�| d k rX|
r5|
Vd }
t | | | | f | f Vd } q�| d k rq| d } n | d k r�| d } |
r�|
Vd }
t | | | | f Vqyt | | | | f | | d f | f V| d } qyWqX W|
r�|
Vd }
x8 | d d � D]& } t d | d f | d f d f Vq Wt d | d f | d f d f Vd S)+Nr r �
0123456789ry Fr zEOF in multi-line stringrx z\
r� z\
rq � � z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.T�
�async�await�def� �\z([{z)]})ry r )ry r ������r� r� r� r� r� r� )r� r� r� ) �string�
ascii_lettersr� rz r] r� r� �STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr} rr r� �IndentationErrorr� �
pseudoprog�spanr� r| �
triple_quoted�endprogs�
single_quotedr� r� r� �OP� ENDMARKER)!ri �lnum�parenlev� continuedZ namechars�numchars�contstr�needcont�contliner� �stashed� async_def�async_def_indent� async_def_nlrf �pos�max�strstart�endprog�endmatchr� �column�
comment_token�nl_pos� pseudomatchrs �spos�eposr �initial�newliner� �indentr r r r [ sp
2
$#$
* $
$�__main__) r r r r r( r) r r! r* r+ r" r# r, r- r$ r% r0 r1 r. r/ r2 r3 r& r' r6 r7 r4 r5 r8 r9 ) r r r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ )Q�
__author__� __credits__r� �re�codecsr r Zlib2to3.pgen2.tokenry r �dir�__all__r� � NameError�strr r r �
Whitespace�Comment�Ignore�Name� Binnumber� Hexnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtras� PseudoToken�list�map�compileZ tokenprogr� Z single3progZ double3progr� r� r r� r� � Exceptionr] ra rg r
rh rk �ASCIIr� r� r� r� r r r^ �sysrz �argv�openri �stdinr r r r �<module> s�
*
8 I �
Mini Shell