File manager - Edit - /opt/saltstack/salt/lib/python3.10/site-packages/charset_normalizer/__pycache__/md.cpython-310.pyc
Back
o ;j�H � @ sf d dl mZ d dlmZ d dlmZmZ ddlmZm Z m Z ddl m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd� d�Z G d d � d e �Z G d d � d e �Z G d d� de �Z G dd� de �Z G dd� de �Z!G dd� de �Z"G dd� de �Z#G dd� de �Z$edd�dee% d ee% d e&fd d ��Z'ed d� "d)d#e%d$e(d%e&d e(fd&d'��Z)d(S )*� )� lru_cache)� getLogger)�List�Optional� )� COMMON_SAFE_ASCII_CHARACTERS�TRACE� UNICODE_SECONDARY_RANGE_KEYWORD)�is_accentuated�is_ascii�is_case_variable�is_cjk� is_emoticon� is_hangul� is_hiragana� is_katakana�is_latin�is_punctuation� is_separator� is_symbol�is_thai�is_unprintable� remove_accent� unicode_rangec @ sP e Zd ZdZdedefdd�Zdeddfdd�Zd d d �Ze de fd d ��Z dS )�MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. � character�returnc C � t �)z@ Determine if given character should be fed in. ��NotImplementedError��selfr � r" �I/opt/saltstack/salt/lib/python3.10/site-packages/charset_normalizer/md.py�eligible$ � zMessDetectorPlugin.eligibleNc C r )z� The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. r r r" r" r# �feed* s zMessDetectorPlugin.feedc C r )zB Permit to reset the plugin to the initial state. r �r! r" r" r# �reset1 r% zMessDetectorPlugin.resetc C r )z� Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. r r' r" r" r# �ratio7 s zMessDetectorPlugin.ratio�r N) �__name__� __module__� __qualname__�__doc__�str�boolr$ r& r( �property�floatr) r" r" r" r# r s r c @ �V e Zd Zddd�Zdedefdd�Zdeddfdd �Zdd d �Ze de fd d ��Z dS )� TooManySymbolOrPunctuationPluginr Nc C s" d| _ d| _d| _d | _d| _d S )Nr F)�_punctuation_count� _symbol_count�_character_count�_last_printable_charZ_frenzy_symbol_in_wordr' r" r" r# �__init__A s z)TooManySymbolOrPunctuationPlugin.__init__r c C � |� � S �N�� isprintabler r" r" r# r$ I � z)TooManySymbolOrPunctuationPlugin.eligiblec C sp | j d7 _ || jkr3|tvr3t|�r | jd7 _n|�� du r3t|�r3t|�du r3| jd7 _|| _d S )Nr F� ) r7 r8 r r r5 �isdigitr r r6 r r" r" r# r& L s � z%TooManySymbolOrPunctuationPlugin.feedc C s d| _ d| _d| _d S �Nr )r5 r7 r6 r' r" r" r# r( ^ � z&TooManySymbolOrPunctuationPlugin.resetc C s0 | j dkrdS | j| j | j }|dkr|S dS )Nr � �333333�?)r7 r5 r6 )r! Zratio_of_punctuationr" r" r# r) c s �z&TooManySymbolOrPunctuationPlugin.ratior* � r+ r, r- r9 r/ r0 r$ r& r( r1 r2 r) r" r" r" r# r4 @ s r4 c @ r3 )�TooManyAccentuatedPluginr Nc C � d| _ d| _d S rA �r7 �_accentuated_countr' r" r" r# r9 p � z!TooManyAccentuatedPlugin.__init__r c C r: r; )�isalphar r" r" r# r$ t r> z!TooManyAccentuatedPlugin.eligiblec C s, | j d7 _ t|�r| jd7 _d S d S �Nr )r7 r rI r r" r" r# r& w s �z TooManyAccentuatedPlugin.feedc C rG rA rH r' r" r" r# r( } rJ z TooManyAccentuatedPlugin.resetc C s4 | j dks | j dk r dS | j| j }|dkr|S dS )Nr � rC gffffff�?rH )r! Zratio_of_accentuationr" r" r# r) � s z TooManyAccentuatedPlugin.ratior* rE r" r" r" r# rF o s rF c @ r3 )�UnprintablePluginr Nc C rG rA )�_unprintable_countr7 r' r" r" r# r9 � rJ zUnprintablePlugin.__init__r c C � dS �NTr" r r" r" r# r$ � � zUnprintablePlugin.eligiblec C s( t |�r | jd7 _| jd7 _d S rL )r rO r7 r r" r" r# r& � s zUnprintablePlugin.feedc C s d| _ d S rA )rO r' r" r" r# r( � s zUnprintablePlugin.resetc C � | j dkrdS | jd | j S )Nr rC rM )r7 rO r' r" r" r# r) � � zUnprintablePlugin.ratior* rE r" r" r" r# rN � s rN c @ r3 )� SuspiciousDuplicateAccentPluginr Nc C � d| _ d| _d | _d S rA ��_successive_countr7 �_last_latin_characterr' r" r" r# r9 � s z(SuspiciousDuplicateAccentPlugin.__init__r c C s |� � ot|�S r; )rK r r r" r" r# r$ � s z(SuspiciousDuplicateAccentPlugin.eligiblec C st | j d7 _ | jd ur5t|�r5t| j�r5|�� r%| j�� r%| jd7 _t|�t| j�kr5| jd7 _|| _d S rL )r7 rY r �isupperrX r r r" r" r# r& � s �� z$SuspiciousDuplicateAccentPlugin.feedc C rV rA rW r' r" r" r# r( � rB z%SuspiciousDuplicateAccentPlugin.resetc C rS )Nr rC r? )r7 rX r' r" r" r# r) � rT z%SuspiciousDuplicateAccentPlugin.ratior* rE r" r" r" r# rU � s rU c @ r3 )�SuspiciousRanger Nc C rV rA )�"_suspicious_successive_range_countr7 �_last_printable_seenr' r" r" r# r9 � rB zSuspiciousRange.__init__r c C r: r; r< r r" r" r# r$ � r> zSuspiciousRange.eligiblec C sx | j d7 _ |�� st|�s|tv rd | _d S | jd u r"|| _d S t| j�}t|�}t||�r7| jd7 _|| _d S rL )r7 �isspacer r r] r � is_suspiciously_successive_ranger\ )r! r �unicode_range_a�unicode_range_br" r" r# r& � s �� zSuspiciousRange.feedc C rV rA )r7 r\ r] r' r" r" r# r( � rB zSuspiciousRange.resetc C s. | j dkrdS | jd | j }|dk rdS |S )Nr rC r? g�������?)r7 r\ )r! Z ratio_of_suspicious_range_usager" r" r# r) � s �zSuspiciousRange.ratior* rE r" r" r" r# r[ � s r[ c @ r3 )�SuperWeirdWordPluginr Nc C s: d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr F� ) � _word_count�_bad_word_count�_foreign_long_count�_is_current_word_bad�_foreign_long_watchr7 �_bad_character_count�_buffer�_buffer_accent_countr' r" r" r# r9 � s z SuperWeirdWordPlugin.__init__r c C rP rQ r" r r" r" r# r$ rR z SuperWeirdWordPlugin.eligiblec C s |� � rH| j|7 _t|�r| jd7 _| jdu rFt|�du s%t|�rFt|�du rFt|�du rFt|�du rFt |�du rFt |�du rFd| _d S | jsMd S |� � sYt |�sYt |�r�| jr�| jd7 _t| j�}| j|7 _|dkr�| j| dkr}d| _t| jd �r�| jd �� r�| jd7 _d| _|dkr�| jr�dd � t| jtd |��D �}d}|r�t|�| d kr�d}|s�| jd7 _d| _| jr�| jd7 _| jt| j�7 _d| _d| _d | _d | _d S |d v�r|�� du �rt|��rd| _| j|7 _d S d S d S d S )Nr FT� g�(\���?���� c S s g | ] \}}|� � r|�qS r" )rZ )�.0�c�ir" r" r# � <listcomp>/ s ��z-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r rD rc > �=�-�_�<�>�~�|)rK rj r rk rh r r r r r r r^ r r rd �lenr7 rg rZ rf �zip�rangere ri r@ r )r! r Z buffer_lengthZcamel_case_dstZprobable_camel_casedr" r" r# r& sr ���� � ��zSuperWeirdWordPlugin.feedc C s4 d| _ d| _d| _d| _d| _d| _d| _d| _d S )Nrc Fr )rj rg rh re rd r7 ri rf r' r" r" r# r( M s zSuperWeirdWordPlugin.resetc C s$ | j dkr | jdkr dS | j| j S )N� r rC )rd rf ri r7 r' r" r" r# r) W s zSuperWeirdWordPlugin.ratior* rE r" r" r" r# rb � s A rb c @ sZ e Zd ZdZddd�Zdedefdd�Zdeddfd d �Zdd d �Z e de fd d��Z dS )�CjkInvalidStopPluginu� GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and can be easily detected. Searching for the overuse of '丅' and '丄'. r Nc C rG rA ��_wrong_stop_count�_cjk_character_countr' r" r" r# r9 e rJ z CjkInvalidStopPlugin.__init__r c C rP rQ r" r r" r" r# r$ i rR z CjkInvalidStopPlugin.eligiblec C s8 |dv r | j d7 _ d S t|�r| jd7 _d S d S )N> u 丄u 丅r )r� r r� r r" r" r# r& l s �zCjkInvalidStopPlugin.feedc C rG rA r r' r" r" r# r( s rJ zCjkInvalidStopPlugin.resetc C s | j dk rdS | j| j S )N� rC )r� r� r' r" r" r# r) w s zCjkInvalidStopPlugin.ratior* ) r+ r, r- r. r9 r/ r0 r$ r& r( r1 r2 r) r" r" r" r# r~ _ s r~ c @ r3 )�ArchaicUpperLowerPluginr Nc C s. d| _ d| _d| _d| _d| _d | _d| _d S )NFr T)�_buf� _character_count_since_last_sep� _successive_upper_lower_count�#_successive_upper_lower_count_finalr7 �_last_alpha_seen�_current_ascii_onlyr' r" r" r# r9 s z ArchaicUpperLowerPlugin.__init__r c C rP rQ r" r r" r" r# r$ � rR z ArchaicUpperLowerPlugin.eligiblec C s$ |� � ot|�}|du }|rC| jdkrC| jdkr+|�� du r+| jdu r+| j| j7 _d| _d| _d | _d| _| j d7 _ d| _d S | jdu rQt |�du rQd| _| jd ur|� � r_| j� � sh|� � r|| j� � r|| jdu rx| jd7 _d| _nd| _nd| _| j d7 _ | jd7 _|| _d S )NFr �@ r Tr? ) rK r r� r@ r� r� r� r� r� r7 r rZ �islower)r! r Z is_concernedZ chunk_sepr" r" r# r& � s@ � �� z ArchaicUpperLowerPlugin.feedc C s. d| _ d| _d| _d| _d | _d| _d| _d S )Nr FT)r7 r� r� r� r� r� r� r' r" r" r# r( � s z ArchaicUpperLowerPlugin.resetc C s | j dkrdS | j| j S )Nr rC )r7 r� r' r" r" r# r) � s z ArchaicUpperLowerPlugin.ratior* rE r" r" r" r# r� ~ s * r� � )�maxsizer` ra r c C sb | du s|du r dS | |krdS d| v rd|v rdS d| v s"d|v r$dS d| v s,d|v r6d| v s4d|v r6dS | � d�|� d�}}|D ]}|tv rJqC||v rQ dS qC| dv |dv }}|s_|rid | v sgd |v ridS |ro|rodS d | v swd |v r�d | v sd |v r�dS | d ks�|d kr�dS d | v s�d |v s�| d v r�|d v r�d | v s�d |v r�dS d| v s�d|v r�dS dS )za Determine if two Unicode range seen next to each other can be considered as suspicious. NTFZLatinZ EmoticonsZ Combining� )�Hiragana�KatakanaZCJKZHangulz Basic Latin)r� r� Z PunctuationZForms)�splitr )r` ra Zkeywords_range_aZkeywords_range_b�elZrange_a_jp_charsZrange_b_jp_charsr" r" r# r_ � sZ �����r_ i 皙�����?F�decoded_sequence�maximum_threshold�debugc C sR dd� t �� D �}t| �d }d}|dk rd}n |dkr d}nd }t| d t|��D ]2\}}|D ] } | �|�r<| �|� q0|d krG|| d ksM||d kr\td d � |D ��}||kr\ nq*|r�td�} | � t d|� d|� d|� �� t| �dkr�| � t d| dd� � �� | � t d| dd� � �� |D ]} | � t | j � d| j � �� q�t |d�S )zw Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. c S s g | ]}|� �qS r" r" )ro Zmd_classr" r" r# rr s �z mess_ratio.<locals>.<listcomp>r rC i � r� r� � � r c s s � | ]}|j V qd S r; )r) )ro �dtr" r" r# � <genexpr>0 s � z mess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=r� zStarting with: Nz Ending with: i�z: � )r �__subclasses__rz r{ r| r$ r& �sumr �logr � __class__r) �round) r� r� r� Z detectors�lengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr �indexZdetector�loggerr� r" r" r# � mess_ratio sN � � ����� r� N)r� F)*� functoolsr �loggingr �typingr r Zconstantr r r �utilsr r r r r r r r r r r r r r r r r r4 rF rN rU r[ rb r~ r� r/ r0 r_ r2 r� r" r" r" r# �<module> sB H"/%4e L�� �F�����
| ver. 1.4 |
Github
|
.
| PHP 8.2.30 | Generation time: 0 |
proxy
|
phpinfo
|
Settings