venv/lib/python3.11/site-packages/pip/_vendor/chardet/utf8prober.py

   1 ######################## BEGIN LICENSE BLOCK ########################
   2 # The Original Code is mozilla.org code.
   3 #
   4 # The Initial Developer of the Original Code is
   5 # Netscape Communications Corporation.
   6 # Portions created by the Initial Developer are Copyright (C) 1998
   7 # the Initial Developer. All Rights Reserved.
   8 #
   9 # Contributor(s):
  10 #   Mark Pilgrim - port to Python
  11 #
  12 # This library is free software; you can redistribute it and/or
  13 # modify it under the terms of the GNU Lesser General Public
  14 # License as published by the Free Software Foundation; either
  15 # version 2.1 of the License, or (at your option) any later version.
  16 #
  17 # This library is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20 # Lesser General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU Lesser General Public
  23 # License along with this library; if not, write to the Free Software
  24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  25 # 02110-1301  USA
  26 ######################### END LICENSE BLOCK #########################
  27
  28 from typing import Union
  29
  30 from .charsetprober import CharSetProber
  31 from .codingstatemachine import CodingStateMachine
  32 from .enums import MachineState, ProbingState
  33 from .mbcssm import UTF8_SM_MODEL
  34
  35
  36 class UTF8Prober(CharSetProber):
  37     ONE_CHAR_PROB = 0.5
  38
  39     def __init__(self) -> None:
  40         super().__init__()
  41         self.coding_sm = CodingStateMachine(UTF8_SM_MODEL)
  42         self._num_mb_chars = 0
  43         self.reset()
  44
  45     def reset(self) -> None:
  46         super().reset()
  47         self.coding_sm.reset()
  48         self._num_mb_chars = 0
  49
  50     @property
  51     def charset_name(self) -> str:
  52         return "utf-8"
  53
  54     @property
  55     def language(self) -> str:
  56         return ""
  57
  58     def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
  59         for c in byte_str:
  60             coding_state = self.coding_sm.next_state(c)
  61             if coding_state == MachineState.ERROR:
  62                 self._state = ProbingState.NOT_ME
  63                 break
  64             if coding_state == MachineState.ITS_ME:
  65                 self._state = ProbingState.FOUND_IT
  66                 break
  67             if coding_state == MachineState.START:
  68                 if self.coding_sm.get_current_charlen() >= 2:
  69                     self._num_mb_chars += 1
  70
  71         if self.state == ProbingState.DETECTING:
  72             if self.get_confidence() > self.SHORTCUT_THRESHOLD:
  73                 self._state = ProbingState.FOUND_IT
  74
  75         return self.state
  76
  77     def get_confidence(self) -> float:
  78         unlike = 0.99
  79         if self._num_mb_chars < 6:
  80             unlike *= self.ONE_CHAR_PROB**self._num_mb_chars
  81             return 1.0 - unlike
  82         return unlike